You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/07/06 15:02:15 UTC
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1347 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1347/
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.update.TestHdfsUpdateLog
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([71DD60B0BD0D738A]:0)
Build Log:
[...truncated 12796 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20170706_124322_0854238295808738570171.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6160.hprof ...
[junit4] Heap dump file created [660046704 bytes in 14.766 secs]
[junit4] <<< JVM J1: EOF ----
[junit4] JVM J1: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20170706_124322_085772138592570646870.syserr
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] WARN: Unhandled exception in event serialization. -> java.lang.OutOfMemoryError: Java heap space
[junit4] <<< JVM J1: EOF ----
[...truncated 835 lines...]
[junit4] Suite: org.apache.solr.update.TestHdfsUpdateLog
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestHdfsUpdateLog_71DD60B0BD0D738A-001/init-core-data-001
[junit4] 2> 956167 WARN (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 956167 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 956169 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 956562 WARN (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 957225 WARN (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 957412 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
[junit4] 2> 957430 WARN (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 957500 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 957546 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_59506_hdfs____f3fkw1/webapp
[junit4] 2> 958299 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:59506
[junit4] 2> 959098 WARN (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 959103 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 959118 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_51001_datanode____m83wxo/webapp
[junit4] 2> 959498 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51001
[junit4] 2> 960284 WARN (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 960286 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 960306 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_51152_datanode____hzyhl/webapp
[junit4] 2> 960862 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51152
[junit4] 2> 961384 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* processReport: from storage DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46 node DatanodeRegistration(127.0.0.1:49950, datanodeUuid=7c56d65d-5c5e-4c4e-b6ef-1172504a9545, infoPort=53005, infoSecurePort=0, ipcPort=47687, storageInfo=lv=-56;cid=testClusterID;nsid=1942765487;c=0), blocks: 0, hasStaleStorage: true, processing time: 2 msecs
[junit4] 2> 961384 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* processReport: from storage DS-808389ac-305f-4349-bd5b-26eb5df476d5 node DatanodeRegistration(127.0.0.1:49950, datanodeUuid=7c56d65d-5c5e-4c4e-b6ef-1172504a9545, infoPort=53005, infoSecurePort=0, ipcPort=47687, storageInfo=lv=-56;cid=testClusterID;nsid=1942765487;c=0), blocks: 0, hasStaleStorage: false, processing time: 1 msecs
[junit4] 2> 961386 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* processReport: from storage DS-d1c676bf-e31b-482f-bf21-cabf5e38a453 node DatanodeRegistration(127.0.0.1:37760, datanodeUuid=d0aa286b-d7cb-4288-a8a2-c667f5e9c420, infoPort=57053, infoSecurePort=0, ipcPort=42785, storageInfo=lv=-56;cid=testClusterID;nsid=1942765487;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 961386 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* processReport: from storage DS-b6f9d0de-9965-431f-806e-b36d32210db3 node DatanodeRegistration(127.0.0.1:37760, datanodeUuid=d0aa286b-d7cb-4288-a8a2-c667f5e9c420, infoPort=57053, infoSecurePort=0, ipcPort=42785, storageInfo=lv=-56;cid=testClusterID;nsid=1942765487;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 962371 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 962372 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 962408 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 962427 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 962514 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 962560 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42fb1a67
[junit4] 2> 962566 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42fb1a67
[junit4] 2> 962566 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42fb1a67
[junit4] 2> 962568 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 962598 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 962623 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 962718 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 962729 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 962730 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42fb1a67
[junit4] 2> 962752 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48967/solr_hdfs_home
[junit4] 2> 962752 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 962752 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 962752 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[hdfs://localhost:48967/solr_hdfs_home/collection1//x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestHdfsUpdateLog_71DD60B0BD0D738A-001/init-core-data-001/]
[junit4] 2> 962754 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48967/solr_hdfs_home/collection1//x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestHdfsUpdateLog_71DD60B0BD0D738A-001/init-core-data-001/snapshot_metadata
[junit4] 2> 962779 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 962779 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 962779 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 963394 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 963402 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48967/solr_hdfs_home/collection1//x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestHdfsUpdateLog_71DD60B0BD0D738A-001/init-core-data-001
[junit4] 2> 963435 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48967/solr_hdfs_home/collection1//x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestHdfsUpdateLog_71DD60B0BD0D738A-001/init-core-data-001/index
[junit4] 2> 963444 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 963444 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 963452 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 963453 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=87.62109375, floorSegmentMB=1.1240234375, forceMergeDeletesPctAllowed=28.427669690135527, segmentsPerTier=3.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 963697 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW]]} size 72
[junit4] 2> 963697 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741825_1001 size 72
[junit4] 2> 964122 WARN (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 964182 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 964182 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=hdfs://localhost:48967/solr/shard1 defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 964182 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 964201 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 964201 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 964204 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4f6644cf
[junit4] 2> 964295 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ec3e484[collection1] main]
[junit4] 2> 964295 WARN (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
[junit4] 2> 964299 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 964301 INFO (searcherExecutor-2770-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6ec3e484[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 964302 INFO (coreLoadExecutor-2769-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572178200288559104
[junit4] 2> 964305 INFO (SUITE-TestHdfsUpdateLog-seed#[71DD60B0BD0D738A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 964342 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testFSThreadSafety
[junit4] 2> 964343 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 964346 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@63c10078[collection1] realtime]
[junit4] 2> 964346 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 3
[junit4] 2> 964347 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1572178200335745024,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 964347 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:1572178200335745024
[junit4] 2> 964385 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 964385 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 964396 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3213eeb0[collection1] main]
[junit4] 2> 964396 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 964399 INFO (searcherExecutor-2770-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3213eeb0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 964399 INFO (TEST-TestHdfsUpdateLog.testFSThreadSafety-seed#[71DD60B0BD0D738A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 51
[junit4] 2> 964457 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[0 (1572178200398659584)]} 0 50
[junit4] 2> 964472 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[1 (1572178200463671296)]} 0 3
[junit4] 2> 964485 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[2 (1572178200478351360)]} 0 2
[junit4] 2> 964497 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[3 (1572178200491982848)]} 0 1
[junit4] 2> 964511 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[4 (1572178200506662912)]} 0 1
[junit4] 2> 964523 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5 (1572178200519245824)]} 0 1
[junit4] 2> 964535 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 964535 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:0
[junit4] 2> 964535 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[6 (1572178200531828736)]} 0 1
[junit4] 2> 964607 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 964608 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 964655 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[7 (1572178200544411648)]} 0 108
[junit4] 2> 964675 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8 (1572178200675483648)]} 0 5
[junit4] 2> 964685 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741830_1006{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 964686 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741830_1006{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 964693 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[9 (1572178200691212288)]} 0 6
[junit4] 2> 964707 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[10 (1572178200709038080)]} 0 3
[junit4] 2> 964720 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[11 (1572178200724766720)]} 0 2
[junit4] 2> 964728 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741831_1007{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 964730 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741831_1007{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 964733 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[12 (1572178200738398208)]} 0 1
[junit4] 2> 964748 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[13 (1572178200750981120)]} 0 4
[junit4] 2> 964758 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741832_1008{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 964760 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741832_1008{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 964762 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[14 (1572178200767758336)]} 0 3
[junit4] 2> 964779 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[15 (1572178200782438400)]} 0 6
[junit4] 2> 964785 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741825_1001 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 964794 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[16 (1572178200800264192)]} 0 4
[junit4] 2> 964810 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[17 (1572178200815992832)]} 0 4
[junit4] 2> 964822 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[18 (1572178200831721472)]} 0 2
[junit4] 2> 964830 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@386d5a6c[collection1] realtime]
[junit4] 2> 964838 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW]]} size 75
[junit4] 2> 964839 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW]]} size 75
[junit4] 2> 964844 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 964845 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 964845 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:0
[junit4] 2> 964846 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[19 (1572178200845352960)]} 0 12
[junit4] 2> 964906 INFO (org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7dfc4197) [ ] BlockStateChange BLOCK* BlockManager: ask 127.0.0.1:37760 to delete [blk_1073741825_1001]
[junit4] 2> 964916 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741833_1009{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 964916 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[20 (1572178200869470208)]} 0 60
[junit4] 2> 964917 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741833_1009{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 964929 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[21 (1572178200943919104)]} 0 2
[junit4] 2> 964941 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[22 (1572178200957550592)]} 0 1
[junit4] 2> 964954 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[23 (1572178200970133504)]} 0 2
[junit4] 2> 964965 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741835_1011{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 964966 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[24 (1572178200983764992)]} 0 1
[junit4] 2> 964968 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741835_1011 size 882
[junit4] 2> 964979 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[25 (1572178200996347904)]} 0 1
[junit4] 2> 964992 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[26 (1572178201009979392)]} 0 1
[junit4] 2> 965015 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[27 (1572178201022562304)]} 0 12
[junit4] 2> 965022 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741836_1012{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965023 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741836_1012{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965029 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[28 (1572178201046679552)]} 0 3
[junit4] 2> 965041 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[29 (1572178201062408192)]} 0 1
[junit4] 2> 965053 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[30 (1572178201074991104)]} 0 1
[junit4] 2> 965067 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741837_1013{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965068 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741837_1013{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965069 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[31 (1572178201087574016)]} 0 4
[junit4] 2> 965080 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[32 (1572178201103302656)]} 0 1
[junit4] 2> 965092 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[33 (1572178201115885568)]} 0 1
[junit4] 2> 965104 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[34 (1572178201128468480)]} 0 1
[junit4] 2> 965116 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[35 (1572178201141051392)]} 0 1
[junit4] 2> 965130 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[36 (1572178201155731456)]} 0 1
[junit4] 2> 965142 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741838_1014{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965144 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[37 (1572178201168314368)]} 0 2
[junit4] 2> 965144 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741838_1014 size 384
[junit4] 2> 965156 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[38 (1572178201181945856)]} 0 1
[junit4] 2> 965168 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[39 (1572178201194528768)]} 0 1
[junit4] 2> 965180 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[40 (1572178201207111680)]} 0 1
[junit4] 2> 965191 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[41 (1572178201219694592)]} 0 1
[junit4] 2> 965203 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[42 (1572178201232277504)]} 0 1
[junit4] 2> 965215 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[43 (1572178201244860416)]} 0 1
[junit4] 2> 965227 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[44 (1572178201257443328)]} 0 1
[junit4] 2> 965229 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741839_1015{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965230 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741839_1015{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965239 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[45 (1572178201270026240)]} 0 1
[junit4] 2> 965252 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[46 (1572178201281560576)]} 0 2
[junit4] 2> 965264 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[47 (1572178201296240640)]} 0 1
[junit4] 2> 965275 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[48 (1572178201307774976)]} 0 1
[junit4] 2> 965299 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[49 (1572178201320357888)]} 0 13
[junit4] 2> 965310 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741840_1016{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 0
[junit4] 2> 965312 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741840_1016 size 795
[junit4] 2> 965321 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[50 (1572178201352863744)]} 0 3
[junit4] 2> 965346 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[51 (1572178201368592384)]} 0 14
[junit4] 2> 965359 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[52 (1572178201395855360)]} 0 1
[junit4] 2> 965371 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[53 (1572178201408438272)]} 0 1
[junit4] 2> 965372 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@21ebc70c[collection1] realtime]
[junit4] 2> 965383 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741829_1005{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW]]} size 75
[junit4] 2> 965384 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741829_1005{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW]]} size 75
[junit4] 2> 965385 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 965385 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 965385 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:0
[junit4] 2> 965386 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[54 (1572178201421021184)]} 0 4
[junit4] 2> 965456 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741841_1017{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965457 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741841_1017{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965485 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[55 (1572178201436749824)]} 0 88
[junit4] 2> 965497 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[56 (1572178201540558848)]} 0 0
[junit4] 2> 965499 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741843_1019{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965501 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741843_1019{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965509 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[57 (1572178201553141760)]} 0 1
[junit4] 2> 965523 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[58 (1572178201567821824)]} 0 1
[junit4] 2> 965524 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741844_1020{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965530 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741844_1020{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965536 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[59 (1572178201580404736)]} 0 1
[junit4] 2> 965547 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[60 (1572178201592987648)]} 0 1
[junit4] 2> 965560 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[61 (1572178201605570560)]} 0 2
[junit4] 2> 965566 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741845_1021{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965567 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741845_1021 size 221
[junit4] 2> 965574 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[62 (1572178201618153472)]} 0 3
[junit4] 2> 965586 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[63 (1572178201633882112)]} 0 1
[junit4] 2> 965601 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[64 (1572178201649610752)]} 0 1
[junit4] 2> 965604 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741846_1022{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965605 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741846_1022{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965613 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[65 (1572178201662193664)]} 0 1
[junit4] 2> 965630 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741847_1023{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965630 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[66 (1572178201677922304)]} 0 3
[junit4] 2> 965631 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741847_1023{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965643 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[67 (1572178201692602368)]} 0 2
[junit4] 2> 965656 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[68 (1572178201707282432)]} 0 1
[junit4] 2> 965661 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741848_1024{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965661 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741848_1024{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965668 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[69 (1572178201719865344)]} 0 1
[junit4] 2> 965669 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741828_1004 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 965684 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[70 (1572178201731399680)]} 0 6
[junit4] 2> 965696 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[71 (1572178201749225472)]} 0 1
[junit4] 2> 965698 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2fdbc252[collection1] realtime]
[junit4] 2> 965710 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741834_1010{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 76
[junit4] 2> 965710 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741834_1010{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 76
[junit4] 2> 965711 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 965711 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 965711 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:0
[junit4] 2> 965712 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[72 (1572178201761808384)]} 0 5
[junit4] 2> 965747 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741849_1025{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965747 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741849_1025{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965774 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[73 (1572178201778585600)]} 0 51
[junit4] 2> 965778 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741851_1027{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965783 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741851_1027 size 221
[junit4] 2> 965786 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[74 (1572178201843597312)]} 0 1
[junit4] 2> 965802 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[75 (1572178201856180224)]} 0 5
[junit4] 2> 965814 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[76 (1572178201871908864)]} 0 1
[junit4] 2> 965825 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[77 (1572178201884491776)]} 0 1
[junit4] 2> 965830 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741852_1028{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 0
[junit4] 2> 965830 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741852_1028{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965837 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[78 (1572178201897074688)]} 0 1
[junit4] 2> 965849 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[79 (1572178201909657600)]} 0 1
[junit4] 2> 965861 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741853_1029{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965862 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[80 (1572178201922240512)]} 0 1
[junit4] 2> 965862 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741853_1029{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965875 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741854_1030{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965875 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[81 (1572178201936920576)]} 0 1
[junit4] 2> 965876 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741854_1030{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 965887 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[82 (1572178201949503488)]} 0 1
[junit4] 2> 965903 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[83 (1572178201962086400)]} 0 4
[junit4] 2> 965915 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[84 (1572178201977815040)]} 0 1
[junit4] 2> 965918 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741855_1031{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965919 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741855_1031{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 965934 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[85 (1572178201991446528)]} 0 7
[junit4] 2> 965945 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[86 (1572178202010320896)]} 0 1
[junit4] 2> 965957 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[87 (1572178202022903808)]} 0 1
[junit4] 2> 965971 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[88 (1572178202035486720)]} 0 3
[junit4] 2> 965974 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741856_1032{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965975 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741856_1032{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 965986 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[89 (1572178202051215360)]} 0 3
[junit4] 2> 966000 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[90 (1572178202066944000)]} 0 2
[junit4] 2> 966001 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741857_1033{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966006 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741857_1033 size 384
[junit4] 2> 966012 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[91 (1572178202080575488)]} 0 1
[junit4] 2> 966028 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[92 (1572178202096304128)]} 0 1
[junit4] 2> 966033 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741858_1034{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966034 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741858_1034 size 421
[junit4] 2> 966039 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[93 (1572178202108887040)]} 0 0
[junit4] 2> 966051 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[94 (1572178202121469952)]} 0 1
[junit4] 2> 966062 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[95 (1572178202133004288)]} 0 1
[junit4] 2> 966074 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[96 (1572178202145587200)]} 0 0
[junit4] 2> 966079 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741859_1035{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966081 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741859_1035{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966086 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[97 (1572178202158170112)]} 0 1
[junit4] 2> 966086 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741833_1009 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966101 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[98 (1572178202169704448)]} 0 4
[junit4] 2> 966110 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2f24f5ef[collection1] realtime]
[junit4] 2> 966131 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741842_1018{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 76
[junit4] 2> 966131 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741842_1018{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 76
[junit4] 2> 966132 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 966132 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 966132 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:0
[junit4] 2> 966178 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741861_1037{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966180 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741861_1037{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966180 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[99 (1572178202192773120)]} 0 62
[junit4] 2> 966192 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[100 (1572178202269319168)]} 0 1
[junit4] 2> 966204 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[101 (1572178202281902080)]} 0 0
[junit4] 2> 966227 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[102 (1572178202306019328)]} 0 1
[junit4] 2> 966231 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741862_1038{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966235 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741862_1038{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966241 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[103 (1572178202318602240)]} 0 3
[junit4] 2> 966255 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[104 (1572178202335379456)]} 0 1
[junit4] 2> 966262 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741863_1039{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966264 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741863_1039{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966266 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[105 (1572178202346913792)]} 0 1
[junit4] 2> 966278 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[106 (1572178202359496704)]} 0 1
[junit4] 2> 966290 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[107 (1572178202372079616)]} 0 1
[junit4] 2> 966293 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741864_1040{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966293 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741864_1040{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966302 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[108 (1572178202384662528)]} 0 1
[junit4] 2> 966313 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[109 (1572178202396196864)]} 0 1
[junit4] 2> 966314 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741865_1041{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW]]} size 0
[junit4] 2> 966314 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741865_1041{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966325 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[110 (1572178202408779776)]} 0 1
[junit4] 2> 966337 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741866_1042{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966337 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[111 (1572178202421362688)]} 0 1
[junit4] 2> 966339 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741866_1042{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966349 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[112 (1572178202433945600)]} 0 1
[junit4] 2> 966356 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741867_1043{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966356 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741867_1043{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966361 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[113 (1572178202446528512)]} 0 1
[junit4] 2> 966362 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741841_1017 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966364 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741831_1007 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966366 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741843_1019 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966368 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741836_1012 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966372 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[114 (1572178202458062848)]} 0 1
[junit4] 2> 966379 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741837_1013 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966381 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741838_1014 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966385 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[115 (1572178202470645760)]} 0 2
[junit4] 2> 966385 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741832_1008 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966389 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741840_1016 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966391 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741845_1021 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966393 INFO (IPC Server handler 7 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741835_1011 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966395 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741830_1006 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966396 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[116 (1572178202483228672)]} 0 0
[junit4] 2> 966397 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741839_1015 127.0.0.1:37760 127.0.0.1:49950
[junit4] 2> 966405 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741846_1022 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966408 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[117 (1572178202495811584)]} 0 1
[junit4] 2> 966442 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[118 (1572178202507345920)]} 0 23
[junit4] 2> 966443 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2f05239a[collection1] realtime]
[junit4] 2> 966448 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741850_1026{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 76
[junit4] 2> 966449 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741850_1026{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 76
[junit4] 2> 966452 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741827_1003 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966453 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741829_1005 127.0.0.1:49950 127.0.0.1:37760
[junit4] 2> 966453 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 966453 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 966453 INFO (commitScheduler-2773-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@31b36a01 commitCommandVersion:0
[junit4] 2> 966471 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[119 (1572178202542997504)]} 0 18
[junit4] 2> 966474 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741869_1045{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966474 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741869_1045{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966482 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[120 (1572178202573406208)]} 0 0
[junit4] 2> 966485 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741870_1046{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966486 INFO (IPC Server handler 1 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741870_1046{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966494 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[121 (1572178202585989120)]} 0 1
[junit4] 2> 966505 INFO (IPC Server handler 3 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741871_1047{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966506 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[122 (1572178202598572032)]} 0 1
[junit4] 2> 966511 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741871_1047 size 1008
[junit4] 2> 966517 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[123 (1572178202610106368)]} 0 1
[junit4] 2> 966525 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741872_1048{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED]]} size 0
[junit4] 2> 966525 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741872_1048{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|FINALIZED], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966534 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[124 (1572178202623737856)]} 0 4
[junit4] 2> 966545 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[125 (1572178202639466496)]} 0 1
[junit4] 2> 966548 INFO (IPC Server handler 6 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741873_1049{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966554 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741873_1049 size 421
[junit4] 2> 966557 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[126 (1572178202652049408)]} 0 0
[junit4] 2> 966567 INFO (IPC Server handler 8 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741874_1050{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966569 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[127 (1572178202663583744)]} 0 1
[junit4] 2> 966569 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741874_1050{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966580 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[128 (1572178202676166656)]} 0 1
[junit4] 2> 966586 INFO (IPC Server handler 0 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741875_1051{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4] 2> 966587 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741875_1051 size 221
[junit4] 2> 966591 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[129 (1572178202688749568)]} 0 0
[junit4] 2> 966603 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[130 (1572178202700283904)]} 0 1
[junit4] 2> 966614 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[131 (1572178202712866816)]} 0 0
[junit4] 2> 966615 INFO (IPC Server handler 5 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741876_1052{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 0
[junit4] 2> 966616 INFO (IPC Server handler 4 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741876_1052{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW], ReplicaUC[[DISK]DS-d1c676bf-e31b-482f-bf21-cabf5e38a453:NORMAL:127.0.0.1:37760|RBW]]} size 0
[junit4] 2> 966626 INFO (Thread-49591) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[132 (1572178202724401152)]} 0 1
[junit4] 2> 966630 INFO (IPC Server handler 2 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37760 is added to blk_1073741877_1053{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-808389ac-305f-4349-bd5b-26eb5df476d5:NORMAL:127.0.0.1:49950|RBW]]} size 0
[junit4] 2> 966630 INFO (IPC Server handler 9 on 48967) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49950 is added to blk_1073741877_1053{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b6f9d0de-9965-431f-806e-b36d32210db3:NORMAL:127.0.0.1:37760|RBW], ReplicaUC[[DISK]DS-bb622c97-8f7c-4beb-9ddf-dafbf719dc46:NORMAL:127.0.0.1:49950|FINALIZED]]} size 0
[junit4
[...truncated too long message...]
-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-8.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20170706_124322_0851540403983220184781.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20170706_124322_0854203761868672364522.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:754: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:267: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:549: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1455: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:985: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
Total time: 327 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No artifacts from Lucene-Solr-NightlyTests-master #1346 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1361 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1361/
2 tests failed.
FAILED: org.apache.solr.update.AutoCommitTest.testMaxTime
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([2C3C29D4212BA24C:B6C85436BFB13E70]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:879)
at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=id:529&qt=&start=0&rows=20&version=2.2
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:872)
... 40 more
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
expected:<2> but was:<3>
Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
at __randomizedtesting.SeedInfo.seed([2C3C29D4212BA24C:A468160E8FD7CFB4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12313 lines...]
[junit4] Suite: org.apache.solr.update.AutoCommitTest
[junit4] 2> 2015974 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_2C3C29D4212BA24C-001/init-core-data-001
[junit4] 2> 2015974 WARN (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 2015974 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2015976 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2015976 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 2015976 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2016011 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2016036 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 2016138 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2016168 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2016171 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2016172 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2016173 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2016197 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2016216 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2016313 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2016320 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 2016320 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2016320 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2016320 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_2C3C29D4212BA24C-001/init-core-data-001/]
[junit4] 2> 2016322 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 2016369 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2016369 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2016370 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2016370 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2016370 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=43, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.297980149086326]
[junit4] 2> 2016371 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@68d3f17f[collection1] main]
[junit4] 2> 2016371 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2016378 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2016408 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2016408 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2016408 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2016408 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2016408 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2016408 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2016408 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2016409 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2016409 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2016409 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2016410 INFO (searcherExecutor-5570-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@68d3f17f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2016411 INFO (coreLoadExecutor-5569-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573787915777998848
[junit4] 2> 2016414 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 2016416 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMaxDocs
[junit4] 2> 2016416 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 2016417 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5028435a[collection1] realtime]
[junit4] 2> 2016417 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 1
[junit4] 2> 2016418 ERROR (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties
[junit4] 2> 2016418 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2016442 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2016463 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2016562 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2016569 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
[junit4] 2> 2016584 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2016584 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_2C3C29D4212BA24C-001/init-core-data-001/]
[junit4] 2> 2016627 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2016627 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2016628 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2563652f[collection1] main]
[junit4] 2> 2016628 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2016629 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2016631 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2016631 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2016632 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573787916009734144
[junit4] 2> 2016632 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2016632 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2016632 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2016632 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2016633 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2016633 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2016633 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2016634 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2016635 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 2016636 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2016638 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2563652f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2016638 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6bbc8f4d[collection1] main]
[junit4] 2> 2016638 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@9a2f148
[junit4] 2> 2016639 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=161673544
[junit4] 2> 2016641 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6bbc8f4d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2016646 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[0 (1573787916023365632)]} 0 1
[junit4] 2> 2016646 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[1 (1573787916024414208)]} 0 1
[junit4] 2> 2016646 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[2 (1573787916024414209)]} 0 1
[junit4] 2> 2016646 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[3 (1573787916024414210)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[4 (1573787916025462784)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[5 (1573787916025462785)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[6 (1573787916025462786)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[7 (1573787916025462787)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[8 (1573787916025462788)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[9 (1573787916025462789)]} 0 2
[junit4] 2> 2016647 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[10 (1573787916025462790)]} 0 3
[junit4] 2> 2016648 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[11 (1573787916026511360)]} 0 3
[junit4] 2> 2016648 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[12 (1573787916026511361)]} 0 3
[junit4] 2> 2016648 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[13 (1573787916026511362)]} 0 3
[junit4] 2> 2016648 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:1&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2016650 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[14 (1573787916028608512)]} 0 5
[junit4] 2> 2016651 INFO (commitScheduler-5578-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2016708 INFO (commitScheduler-5578-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c3402c4[collection1] main]
[junit4] 2> 2016710 INFO (searcherExecutor-5575-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7c3402c4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):c15)))}
[junit4] 2> 2016710 INFO (commitScheduler-5578-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2016901 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[15 (1573787916290752512)]} 0 256
[junit4] 2> 2016901 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:14&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2016902 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:15&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2016902 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxDocs
[junit4] 2> 2016908 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testMaxTime
[junit4] 2> 2016908 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 2016910 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33d24e0d[collection1] realtime]
[junit4] 2> 2016910 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 2
[junit4] 2> 2016910 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties
[junit4] 2> 2016911 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2016962 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2016980 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2017062 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2017069 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
[junit4] 2> 2017079 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2017079 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_2C3C29D4212BA24C-001/init-core-data-001/]
[junit4] 2> 2017121 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2017121 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2017122 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6532f7a5[collection1] main]
[junit4] 2> 2017122 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2017123 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2017125 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2017128 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=14.400390625, floorSegmentMB=2.10546875, forceMergeDeletesPctAllowed=20.251443402924643, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2017128 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2017128 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2017128 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2017129 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2017131 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6532f7a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2017131 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4afead73[collection1] main]
[junit4] 2> 2017132 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@2be76017
[junit4] 2> 2017132 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=736583703
[junit4] 2> 2017135 INFO (searcherExecutor-5579-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4afead73[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2017135 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1573787916536119296)]} 0 1
[junit4] 2> 2017136 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2017136 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1573787916538216448)]} 0 2
[junit4] 2> 2017136 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2017136 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] 2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] 2> <response>
[junit4] 2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] 2> </response>
[junit4] 2>
[junit4] 2> request was:q=id:529&qt=&start=0&rows=20&version=2.2
[junit4] 2> 2017137 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=id:529&qt=&start=0&rows=20&version=2.2:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] 2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] 2> <response>
[junit4] 2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] 2> </response>
[junit4] 2>
[junit4] 2> request was:q=id:529&qt=&start=0&rows=20&version=2.2
[junit4] 2> at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:872)
[junit4] 2> at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2017137 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxTime
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=AutoCommitTest -Dtests.method=testMaxTime -Dtests.seed=2C3C29D4212BA24C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=et-EE -Dtests.timezone=Asia/Almaty -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.24s J2 | AutoCommitTest.testMaxTime <<<
[junit4] > Throwable #1: java.lang.RuntimeException: Exception during query
[junit4] > at __randomizedtesting.SeedInfo.seed([2C3C29D4212BA24C:B6C85436BFB13E70]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:879)
[junit4] > at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] > xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] > <response>
[junit4] > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] > </response>
[junit4] > request was:q=id:529&qt=&start=0&rows=20&version=2.2
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:872)
[junit4] > ... 40 more
[junit4] 2> 2017152 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testCommitWithin
[junit4] 2> 2017152 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 2017152 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3fc61cfc[collection1] realtime]
[junit4] 2> 2017152 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 0
[junit4] 2> 2017152 ERROR (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties
[junit4] 2> 2017153 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2017193 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2017212 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2017317 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2017324 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
[junit4] 2> 2017338 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f1cb5fd
[junit4] 2> 2017338 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_2C3C29D4212BA24C-001/init-core-data-001/]
[junit4] 2> 2017397 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2017397 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2017399 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@575f8cec[collection1] main]
[junit4] 2> 2017399 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2017400 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2017402 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2017405 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2017405 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2017405 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2017405 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2017405 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2017405 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2017406 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1032781089, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2017406 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2017406 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2017406 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2017408 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@575f8cec[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2017408 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2017408 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@51996347[collection1] main]
[junit4] 2> 2017409 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@7e95b77b
[junit4] 2> 2017411 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@51996347[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2017411 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=2123741051
[junit4] 2> 2017420 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1573787916831817728)]} 0 4
[junit4] 2> 2017420 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2021416 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2021470 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2c22fdde[collection1] main]
[junit4] 2> 2021470 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2021472 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2c22fdde[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(8.0.0):c1)))}
[junit4] 2> 2021673 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1573787921294557184)]} 0 4257
[junit4] 2> 2021673 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2021674 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[529 (-1573787921295605760)]} 0 4258
[junit4] 2> 2021674 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2023674 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2023680 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d6dbb0a[collection1] main]
[junit4] 2> 2023680 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2023681 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3d6dbb0a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(8.0.0):c1)))}
[junit4] 2> 2023925 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2023925 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2023926 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923658047488)]} 0 6510
[junit4] 2> 2023926 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923658047489)]} 0 6510
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096064)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096065)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096066)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096067)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096068)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096069)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096070)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1573787923659096071)]} 0 6511
[junit4] 2> 2023927 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144640)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144641)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144642)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144643)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144644)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144645)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144646)]} 0 6512
[junit4] 2> 2023928 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923660144647)]} 0 6512
[junit4] 2> 2023930 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923662241792)]} 0 6514
[junit4] 2> 2023930 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1573787923662241793)]} 0 6514
[junit4] 2> 2023930 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2024928 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2024932 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6554bf[collection1] main]
[junit4] 2> 2024932 INFO (commitScheduler-5586-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2024935 INFO (searcherExecutor-5583-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6554bf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(8.0.0):c10/9:delGen=1)))}
[junit4] 2> 2025181 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2025182 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2025182 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2C3C29D4212BA24C]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testCommitWithin
[junit4] 2> 2025182 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 2025182 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1287937912
[junit4] 2> 2025183 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2025184 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2025186 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2025186 INFO (SUITE-AutoCommitTest-seed#[2C3C29D4212BA24C]-worker) [ x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2025187 INFO (coreCloseExecutor-5587-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@510bd554
[junit4] 2> 2025187 INFO (coreCloseExecutor-5587-thread-1-processing-x:collection1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
[junit4] 2> 2025187 INFO (coreCloseExecutor-5587-thread-1-processing-x:collection1) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@67f67d80 commitCommandVersion:0
[junit4] 2> 2025188 INFO (coreCloseExecutor-5587-thread-1-processing-x:collection1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1359730004
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_2C3C29D4212BA24C-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=BlockTreeOrds(blocksize=128), subject=BlockTreeOrds(blocksize=128), intDefault=Lucene50(blocksize=128), id_i1=Lucene50(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), range_facet_i_dv=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_l=BlockTreeOrds(blocksize=128), timestamp=BlockTreeOrds(blocksize=128)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), _version_=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Lucene70), intDvoDefault=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=2008, maxMBSortInHeap=7.036702183003561, sim=RandomSimilarity(queryNorm=false): {}, locale=et-EE, timezone=Asia/Almaty
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=133181520,total=517996544
[junit4] 2> NOTE: All tests run in this JVM: [ExitableDirectoryReaderTest, StatsReloadRaceTest, SolrMetricReporterTest, TestSuggestSpellingConverter, ClusterStateTest, SolrCoreCheckLockOnStartupTest, SolrInfoBeanTest, TestNestedDocsSort, TestCollationFieldDocValues, SpellPossibilityIteratorTest, CursorMarkTest, TestUtils, TestRawTransformer, TestConfigsApi, TestRecovery, HdfsBasicDistributedZk2Test, TestQuerySenderListener, FastVectorHighlighterTest, TestMinMaxOnMultiValuedField, SearchHandlerTest, SimplePostToolTest, TestSchemaNameResource, CleanupOldIndexTest, ConfigureRecoveryStrategyTest, HdfsSyncSliceTest, TestLegacyNumericRangeQueryBuilder, TestTolerantUpdateProcessorRandomCloud, TestFieldCacheReopen, RollingRestartTest, TestSSLRandomization, RecoveryZkTest, ReplicaListTransformerTest, TestXmlQParserPlugin, TestCollectionAPI, DeleteReplicaTest, TestSolrConfigHandler, ImplicitSnitchTest, HdfsChaosMonkeySafeLeaderTest, ResponseHeaderTest, DeleteInactiveReplicaTest, TestAnalyzedSuggestions, TestRandomCollapseQParserPlugin, TestBlobHandler, TestCustomSort, TestSizeLimitedDistributedMap, TestGroupingSearch, TestGraphTermsQParserPlugin, TestQueryUtils, TermsComponentTest, DataDrivenBlockJoinTest, PKIAuthenticationIntegrationTest, TestRandomFlRTGCloud, DateMathParserTest, TestIndexSearcher, ResponseLogComponentTest, TestDeleteCollectionOnDownNodes, MetricsConfigTest, TestConfigSetsAPIExclusivity, BlockDirectoryTest, HdfsUnloadDistributedZkTest, TestSurroundQueryParser, PolyFieldTest, NumericFieldsTest, TestSolrCLIRunExample, DistributedExpandComponentTest, OverriddenZkACLAndCredentialsProvidersTest, TestFieldCacheVsDocValues, TestLegacyFieldReuse, TestFastWriter, UnloadDistributedZkTest, TestLockTree, BadIndexSchemaTest, SignatureUpdateProcessorFactoryTest, PropertiesRequestHandlerTest, TestCollapseQParserPlugin, MetricUtilsTest, DocumentBuilderTest, TestManagedSchemaAPI, TestUseDocValuesAsStored2, TestTrackingShardHandlerFactory, TestFieldResource, CdcrReplicationHandlerTest, URLClassifyProcessorTest, TestZkChroot, TestSubQueryTransformer, TestManagedStopFilterFactory, PreAnalyzedFieldTest, TestTrie, TestStressUserVersions, BasicDistributedZk2Test, TestRuleBasedAuthorizationPlugin, TestLeaderElectionZkExpiry, SolrTestCaseJ4Test, StatsComponentTest, TolerantUpdateProcessorTest, SuggesterFSTTest, TestImpersonationWithHadoopAuth, SimpleFacetsTest, TestSort, SortByFunctionTest, TestExportWriter, TestPayloadCheckQParserPlugin, TestCloudPseudoReturnFields, CopyFieldTest, V2StandaloneTest, HdfsLockFactoryTest, MinimalSchemaTest, TestLRUStatsCache, TestLFUCache, SolrGraphiteReporterTest, BJQParserTest, NotRequiredUniqueKeyTest, WrapperMergePolicyFactoryTest, ReturnFieldsTest, FullSolrCloudDistribCmdsTest, LeaderElectionTest, ZkSolrClientTest, AutoCommitTest]
[junit4] Completed [330/727 (1!)] on J2 in 9.23s, 3 tests, 1 error <<< FAILURES!
[...truncated 630 lines...]
[junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
[junit4] 2> 2783716 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2C3C29D4212BA24C-001/init-core-data-001
[junit4] 2> 2783716 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=17 numCloses=17
[junit4] 2> 2783717 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2783718 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 2783718 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2783778 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2783807 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2783810 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2783845 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_36479_hdfs____6pvxbx/webapp
[junit4] 2> 2784241 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36479
[junit4] 2> 2784403 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2784404 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2784432 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37324_datanode____.togxjc/webapp
[junit4] 2> 2784886 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37324
[junit4] 2> 2785069 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2785070 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2785094 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40397_datanode____.la28lg/webapp
[junit4] 2> 2785256 INFO (IPC Server handler 5 on 48360) [ ] BlockStateChange BLOCK* processReport: from storage DS-47313cc7-9f53-49e2-9620-4d6b4b288c74 node DatanodeRegistration(127.0.0.1:45915, datanodeUuid=01e0fd26-1b7a-4d5e-b913-607156304e12, infoPort=54809, infoSecurePort=0, ipcPort=44386, storageInfo=lv=-56;cid=testClusterID;nsid=1193686392;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 2785257 INFO (IPC Server handler 5 on 48360) [ ] BlockStateChange BLOCK* processReport: from storage DS-ca27ec8f-f073-4ca6-8cae-5c2272cd12b0 node DatanodeRegistration(127.0.0.1:45915, datanodeUuid=01e0fd26-1b7a-4d5e-b913-607156304e12, infoPort=54809, infoSecurePort=0, ipcPort=44386, storageInfo=lv=-56;cid=testClusterID;nsid=1193686392;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2785528 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C3C29D4212BA24C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40397
[junit4] 2> 2785695 INFO (IPC Server handler 7 on 48360) [ ] BlockStateChange BLOCK* processReport: from storage DS-1d635e1f-aa26-4d42-9a61-52d26e34be5b node DatanodeRegistration(127.0.0.1:50978, datanodeUuid=8a113caf-0cc4-413d-bc2a-da2854c123b5, infoPort=54726, infoSecurePort=0, ipcPort=56844, storageInfo=lv=-56;cid=testClusterID;nsid=1193686392;c=0), blocks: 0, hasStaleStorage: true, processing time: 2 msecs
[junit4] 2> 2785695 INFO (IPC Server handler 7 on 48360) [ ] BlockStateChange BLOCK* processReport: from storage DS-80ee90d8-772d-429c-b2a5-2da14ade305b node DatanodeRegistration(127.0.0.1:50978, datanodeUuid=8a113caf-0cc4-413d-bc2a-da2854c123b5, infoPort=54726, infoSecurePort=0, ipcPort=56844, storageInfo=lv=-56;cid=testClusterID;nsid=1193686392;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2785899 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2785900 INFO (Thread-86077) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2785900 INFO (Thread-86077) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2785908 ERROR (Thread-86077) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2786000 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.ZkTestServer start zk server on port:37961
[junit4] 2> 2786027 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2786028 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2786029 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2786030 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2786032 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2786034 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2786038 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2786039 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2786040 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2786042 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2786044 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2786046 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 2786159 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6db50e59{/,null,AVAILABLE}
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2f9ccb7a{HTTP/1.1,[http/1.1]}{127.0.0.1:57783}
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.e.j.s.Server Started @2788409ms
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, hostContext=/, hostPort=57783, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2C3C29D4212BA24C-001/control-001/cores}
[junit4] 2> 2786161 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2786161 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-24T07:38:04.771Z
[junit4] 2> 2786176 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2786177 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2C3C29D4212BA24C-001/control-001/solr.xml
[junit4] 2> 2786183 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1ce723f9, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2786189 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37961/solr
[junit4] 2> 2786362 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2786364 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57783_
[junit4] 2> 2786366 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.c.Overseer Overseer (id=98361795183247364-127.0.0.1:57783_-n_0000000000) starting
[junit4] 2> 2786383 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57783_
[junit4] 2> 2786384 INFO (zkCallback-2488-thread-1-processing-n:127.0.0.1:57783_) [n:127.0.0.1:57783_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2786436 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ce723f9
[junit4] 2> 2786442 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ce723f9
[junit4] 2> 2786442 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ce723f9
[junit4] 2> 2786443 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [n:127.0.0.1:57783_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2C3C29D4212BA24C-001/control-001/cores
[junit4] 2> 2786465 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2786467 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[2C3C29D4212BA24C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37961/solr ready
[junit4] 2> 2786471 INFO (qtp1556134548-100132) [n:127.0.0.1:57783_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:57783_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2786474 INFO (OverseerThreadFactory-7313-thread-1-processing-n:127.0.0.1:57783_) [n:127.0.0.1:57783_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2786580 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2786581 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2786689 INFO (zkCallback-2488-thread-1-processing-n:127.0.0.1:57783_) [n:127.0.0.1:57783_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2787621 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2787647 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2787764 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2787791 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2787792 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ce723f9
[junit4] 2> 2787792 INFO (qtp1556134548-100134) [n:127.0.0.1:57783_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFa
[...truncated too long message...]
at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2C3C29D4212BA24C-001
[junit4] 2> Jul 24, 2017 7:39:23 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=ja-JP-u-ca-japanese-x-lvariant-JP, timezone=Indian/Maldives
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=2,free=234973488,total=521666560
[junit4] 2> NOTE: All tests run in this JVM: [CoreSorterTest, TestConfigReload, TestAuthorizationFramework, CdcrBootstrapTest, HdfsDirectoryFactoryTest, EchoParamsTest, TestTestInjection, TestJsonFacetRefinement, ManagedSchemaRoundRobinCloudTest, QueryElevationComponentTest, BasicAuthIntegrationTest, TestBinaryField, TestFastOutputStream, SolrJmxReporterTest, TestReplicationHandler, TestHashQParserPlugin, AutoScalingHandlerTest, TestConfigOverlay, DistributedQueryComponentCustomSortTest, TestSolrQueryParser, UpdateParamsTest, MBeansHandlerTest, ParsingFieldUpdateProcessorsTest, TestMacros, BasicZkTest, HdfsRecoveryZkTest, SortSpecParsingTest, TestStressCloudBlindAtomicUpdates, WordBreakSolrSpellCheckerTest, ReplicationFactorTest, TestRandomRequestDistribution, OverseerStatusTest, CoreAdminCreateDiscoverTest, TestSolr4Spatial, TestCloudJSONFacetJoinDomain, CollectionTooManyReplicasTest, TestDynamicLoading, ZkStateReaderTest, HdfsBasicDistributedZkTest, TestExactSharedStatsCache, CloudExitableDirectoryReaderTest, ConjunctionSolrSpellCheckerTest, TestFieldSortValues, TestQuerySenderNoQuery, TestOmitPositions, TestRequestForwarding, TestDistributedSearch, UUIDFieldTest, TestExtendedDismaxParser, DebugComponentTest, BufferStoreTest, TestIBSimilarityFactory, TestMiniSolrCloudClusterSSL, FieldMutatingUpdateProcessorTest, AnalyticsMergeStrategyTest, InfoHandlerTest, TestCollectionsAPIViaSolrCloudCluster, CheckHdfsIndexTest, TestCloudRecovery, SpellCheckCollatorTest, HdfsThreadLeakTest, TestSmileRequest, TestSchemaResource, TestFieldTypeCollectionResource, SolrPluginUtilsTest, ShufflingReplicaListTransformerTest, TestHttpShardHandlerFactory, TestReqParamsAPI, MigrateRouteKeyTest, SuggesterWFSTTest, JavabinLoaderTest, TemplateUpdateProcessorTest, TlogReplayBufferedWhileIndexingTest, TestHdfsCloudBackupRestore, TestPHPSerializedResponseWriter, TestIndexingPerformance, RequestLoggingTest, TestManagedSchemaThreadSafety, HLLUtilTest, TestFieldCacheWithThreads, OpenExchangeRatesOrgProviderTest, SynonymTokenizerTest, DistributedFacetPivotSmallAdvancedTest, TestWriterPerf, AssignTest, ClassificationUpdateProcessorFactoryTest, TestTlogReplica, SolrShardReporterTest, TestCSVResponseWriter, TestSerializedLuceneMatchVersion, TestJoin, TestReloadDeadlock, ConnectionReuseTest, TestLMDirichletSimilarityFactory, TestDistribDocBasedVersion, OutputWriterTest, SecurityConfHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestSha256AuthenticationProvider, TestSolrFieldCacheBean, TestLegacyNumericUtils, SolrCoreMetricManagerTest, TestComponentsName, TestAuthenticationFramework, TestPushWriter, BinaryUpdateRequestHandlerTest, SpellingQueryConverterTest, CursorPagingTest, TestDFISimilarityFactory, LargeFieldTest, DateRangeFieldTest, TestStressLucene, TestDynamicFieldResource, DistributedSpellCheckComponentTest, TestHalfAndHalfDocValues, TestSystemIdResolver, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, TestDocumentBuilder, ZkNodePropsTest, SystemInfoHandlerTest, CircularListTest, DistributedIntervalFacetingTest, BaseCdcrDistributedZkTest, CdcrRequestHandlerTest, CollectionReloadTest, CollectionStateFormat2Test, CollectionsAPISolrJTest, DistribCursorPagingTest, ForceLeaderTest, LeaderInitiatedRecoveryOnShardRestartTest, MissingSegmentRecoveryTest, MoveReplicaHDFSUlogDirTest, NodeMutatorTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, PeerSyncReplicationTest, RecoveryAfterSoftCommitTest, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [536/727 (2!)] on J0 in 90.95s, 1 test, 1 failure <<< FAILURES!
[...truncated 43879 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1360 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1360/
5 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([2C9DFF6E29E6CD07:A4C9C0B4871AA0FF]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest: 1) Thread[id=8280, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=8281, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:
1) Thread[id=8280, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=8281, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([2C9DFF6E29E6CD07]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=8280, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.security.AccessController.doPrivileged(Native Method) at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114) at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99) at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.write(PrintStream.java:482) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) at java.io.PrintStream.newLine(PrintStream.java:546) at java.io.PrintStream.println(PrintStream.java:824) at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748) at java.lang.Throwable.printStackTrace(Throwable.java:658) at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121) 2) Thread[id=8281, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.io.PrintStream.println(PrintStream.java:805) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=8280, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.security.AccessController.doPrivileged(Native Method)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.write(PrintStream.java:482)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
at java.io.PrintStream.newLine(PrintStream.java:546)
at java.io.PrintStream.println(PrintStream.java:824)
at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
at java.lang.Throwable.printStackTrace(Throwable.java:658)
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
2) Thread[id=8281, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.io.PrintStream.println(PrintStream.java:805)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
at __randomizedtesting.SeedInfo.seed([2C9DFF6E29E6CD07]:0)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
Error Message:
expected:<[{indexVersion=1500734086443,generation=2,filelist=[_f6.fdt, _f6.fdx, _f6.fnm, _f6.nvd, _f6.nvm, _f6.si, _f6_LuceneVarGapDocFreqInterval_0.doc, _f6_LuceneVarGapDocFreqInterval_0.tib, _f6_LuceneVarGapDocFreqInterval_0.tiv, _f7.cfe, _f7.cfs, _f7.si, _f8.cfe, _f8.cfs, _f8.si, _f9.cfe, _f9.cfs, _f9.si, _fa.cfe, _fa.cfs, _fa.si, _fb.cfe, _fb.cfs, _fb.si, _fc.cfe, _fc.cfs, _fc.si, _fd.cfe, _fd.cfs, _fd.si, _fe.cfe, _fe.cfs, _fe.si, _ff.cfe, _ff.cfs, _ff.si, _fg.cfe, _fg.cfs, _fg.si, _fh.cfe, _fh.cfs, _fh.si, _fi.cfe, _fi.cfs, _fi.si, _fj.cfe, _fj.cfs, _fj.si, _fk.cfe, _fk.cfs, _fk.si, _fl.cfe, _fl.cfs, _fl.si, _fm.cfe, _fm.cfs, _fm.si, _fn.cfe, _fn.cfs, _fn.si, _fo.cfe, _fo.cfs, _fo.si, _fp.cfe, _fp.cfs, _fp.si, _fq.cfe, _fq.cfs, _fq.si, segments_2]}]> but was:<[{indexVersion=1500734086443,generation=2,filelist=[_f6.fdt, _f6.fdx, _f6.fnm, _f6.nvd, _f6.nvm, _f6.si, _f6_LuceneVarGapDocFreqInterval_0.doc, _f6_LuceneVarGapDocFreqInterval_0.tib, _f6_LuceneVarGapDocFreqInterval_0.tiv, _f7.cfe, _f7.cfs, _f7.si, _f8.cfe, _f8.cfs, _f8.si, _f9.cfe, _f9.cfs, _f9.si, _fa.cfe, _fa.cfs, _fa.si, _fb.cfe, _fb.cfs, _fb.si, _fc.cfe, _fc.cfs, _fc.si, _fd.cfe, _fd.cfs, _fd.si, _fe.cfe, _fe.cfs, _fe.si, _ff.cfe, _ff.cfs, _ff.si, _fg.cfe, _fg.cfs, _fg.si, _fh.cfe, _fh.cfs, _fh.si, _fi.cfe, _fi.cfs, _fi.si, _fj.cfe, _fj.cfs, _fj.si, _fk.cfe, _fk.cfs, _fk.si, _fl.cfe, _fl.cfs, _fl.si, _fm.cfe, _fm.cfs, _fm.si, _fn.cfe, _fn.cfs, _fn.si, _fo.cfe, _fo.cfs, _fo.si, _fp.cfe, _fp.cfs, _fp.si, _fq.cfe, _fq.cfs, _fq.si, segments_2]}, {indexVersion=1500734086443,generation=3,filelist=[_fr.fdt, _fr.fdx, _fr.fnm, _fr.nvd, _fr.nvm, _fr.si, _fr_LuceneVarGapDocFreqInterval_0.doc, _fr_LuceneVarGapDocFreqInterval_0.tib, _fr_LuceneVarGapDocFreqInterval_0.tiv, segments_3]}]>
Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1500734086443,generation=2,filelist=[_f6.fdt, _f6.fdx, _f6.fnm, _f6.nvd, _f6.nvm, _f6.si, _f6_LuceneVarGapDocFreqInterval_0.doc, _f6_LuceneVarGapDocFreqInterval_0.tib, _f6_LuceneVarGapDocFreqInterval_0.tiv, _f7.cfe, _f7.cfs, _f7.si, _f8.cfe, _f8.cfs, _f8.si, _f9.cfe, _f9.cfs, _f9.si, _fa.cfe, _fa.cfs, _fa.si, _fb.cfe, _fb.cfs, _fb.si, _fc.cfe, _fc.cfs, _fc.si, _fd.cfe, _fd.cfs, _fd.si, _fe.cfe, _fe.cfs, _fe.si, _ff.cfe, _ff.cfs, _ff.si, _fg.cfe, _fg.cfs, _fg.si, _fh.cfe, _fh.cfs, _fh.si, _fi.cfe, _fi.cfs, _fi.si, _fj.cfe, _fj.cfs, _fj.si, _fk.cfe, _fk.cfs, _fk.si, _fl.cfe, _fl.cfs, _fl.si, _fm.cfe, _fm.cfs, _fm.si, _fn.cfe, _fn.cfs, _fn.si, _fo.cfe, _fo.cfs, _fo.si, _fp.cfe, _fp.cfs, _fp.si, _fq.cfe, _fq.cfs, _fq.si, segments_2]}]> but was:<[{indexVersion=1500734086443,generation=2,filelist=[_f6.fdt, _f6.fdx, _f6.fnm, _f6.nvd, _f6.nvm, _f6.si, _f6_LuceneVarGapDocFreqInterval_0.doc, _f6_LuceneVarGapDocFreqInterval_0.tib, _f6_LuceneVarGapDocFreqInterval_0.tiv, _f7.cfe, _f7.cfs, _f7.si, _f8.cfe, _f8.cfs, _f8.si, _f9.cfe, _f9.cfs, _f9.si, _fa.cfe, _fa.cfs, _fa.si, _fb.cfe, _fb.cfs, _fb.si, _fc.cfe, _fc.cfs, _fc.si, _fd.cfe, _fd.cfs, _fd.si, _fe.cfe, _fe.cfs, _fe.si, _ff.cfe, _ff.cfs, _ff.si, _fg.cfe, _fg.cfs, _fg.si, _fh.cfe, _fh.cfs, _fh.si, _fi.cfe, _fi.cfs, _fi.si, _fj.cfe, _fj.cfs, _fj.si, _fk.cfe, _fk.cfs, _fk.si, _fl.cfe, _fl.cfs, _fl.si, _fm.cfe, _fm.cfs, _fm.si, _fn.cfe, _fn.cfs, _fn.si, _fo.cfe, _fo.cfs, _fo.si, _fp.cfe, _fp.cfs, _fp.si, _fq.cfe, _fq.cfs, _fq.si, segments_2]}, {indexVersion=1500734086443,generation=3,filelist=[_fr.fdt, _fr.fdx, _fr.fnm, _fr.nvd, _fr.nvm, _fr.si, _fr_LuceneVarGapDocFreqInterval_0.doc, _fr_LuceneVarGapDocFreqInterval_0.tib, _fr_LuceneVarGapDocFreqInterval_0.tiv, segments_3]}]>
at __randomizedtesting.SeedInfo.seed([2C9DFF6E29E6CD07:94AE45E59AEC304]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([2C9DFF6E29E6CD07:A4C9C0B4871AA0FF]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:156)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11519 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> 617301 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/init-core-data-001
[junit4] 2> 617306 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=13 numCloses=13
[junit4] 2> 617306 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 617308 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 617308 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 617385 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 617408 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 617420 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 617444 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_42577_hdfs____ne5npd/webapp
[junit4] 2> 617937 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42577
[junit4] 2> 618280 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 618281 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 618300 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_52896_datanode____.sc0a69/webapp
[junit4] 2> 618942 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:52896
[junit4] 2> 619357 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 619358 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 619427 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_56843_datanode____.5qt1nn/webapp
[junit4] 2> 619537 INFO (IPC Server handler 1 on 36700) [ ] BlockStateChange BLOCK* processReport: from storage DS-bf1ea655-26f7-49cc-8bae-6ca8f9e0893c node DatanodeRegistration(127.0.0.1:60774, datanodeUuid=8e80e510-5f32-466d-8ace-a32ae5773536, infoPort=52724, infoSecurePort=0, ipcPort=38542, storageInfo=lv=-56;cid=testClusterID;nsid=598110306;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 619537 INFO (IPC Server handler 1 on 36700) [ ] BlockStateChange BLOCK* processReport: from storage DS-9ee29c27-84a0-48dc-88c2-cfeda54ca458 node DatanodeRegistration(127.0.0.1:60774, datanodeUuid=8e80e510-5f32-466d-8ace-a32ae5773536, infoPort=52724, infoSecurePort=0, ipcPort=38542, storageInfo=lv=-56;cid=testClusterID;nsid=598110306;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 620260 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:56843
[junit4] 2> 620981 INFO (IPC Server handler 8 on 36700) [ ] BlockStateChange BLOCK* processReport: from storage DS-c134b49c-5a84-4302-bfb3-e2acecd87a90 node DatanodeRegistration(127.0.0.1:44434, datanodeUuid=ec8c1de4-197b-44e4-bae3-08d283e02ead, infoPort=40880, infoSecurePort=0, ipcPort=39282, storageInfo=lv=-56;cid=testClusterID;nsid=598110306;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 620981 INFO (IPC Server handler 8 on 36700) [ ] BlockStateChange BLOCK* processReport: from storage DS-d4ba4116-dc10-4d04-b2c1-862ced4dee34 node DatanodeRegistration(127.0.0.1:44434, datanodeUuid=ec8c1de4-197b-44e4-bae3-08d283e02ead, infoPort=40880, infoSecurePort=0, ipcPort=39282, storageInfo=lv=-56;cid=testClusterID;nsid=598110306;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 621652 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 621662 INFO (Thread-3220) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 621662 INFO (Thread-3220) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 621691 ERROR (Thread-3220) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 621762 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.ZkTestServer start zk server on port:47214
[junit4] 2> 621841 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 621845 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 621853 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 621861 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 621864 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 621875 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 621884 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 621892 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 621898 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 621902 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 621906 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 621912 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 622147 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 622148 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e618250{/,null,AVAILABLE}
[junit4] 2> 622148 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@57298fbf{HTTP/1.1,[http/1.1]}{127.0.0.1:41392}
[junit4] 2> 622148 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.Server Started @624330ms
[junit4] 2> 622148 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:36700/hdfs__localhost_36700__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=41392, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/control-001/cores}
[junit4] 2> 622150 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 622150 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 622150 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 622150 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 622150 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-22T14:28:17.961Z
[junit4] 2> 622153 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 622153 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/control-001/solr.xml
[junit4] 2> 622157 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 622181 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47214/solr
[junit4] 2> 622392 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 622392 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41392_
[junit4] 2> 622393 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.c.Overseer Overseer (id=98352083599425540-127.0.0.1:41392_-n_0000000000) starting
[junit4] 2> 622478 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41392_
[junit4] 2> 622479 INFO (zkCallback-1078-thread-1-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 622731 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 622764 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 622764 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 622765 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:41392_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/control-001/cores
[junit4] 2> 622869 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 622869 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:47214/solr ready
[junit4] 2> 622876 INFO (qtp1666203318-7924) [n:127.0.0.1:41392_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:41392_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 622896 INFO (OverseerThreadFactory-2282-thread-1-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 623008 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 623009 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 623114 INFO (zkCallback-1078-thread-1-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 624070 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 624105 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 624197 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 624238 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 624239 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 624239 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:36700/solr_hdfs_home
[junit4] 2> 624239 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 624239 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 624239 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:36700/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 624240 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 624246 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 624247 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 624247 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 624257 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 624264 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 624288 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 624303 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 624303 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 624303 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 624337 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 624338 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=50, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1339697012392608]
[junit4] 2> 624401 INFO (IPC Server handler 0 on 36700) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44434 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bf1ea655-26f7-49cc-8bae-6ca8f9e0893c:NORMAL:127.0.0.1:60774|RBW], ReplicaUC[[DISK]DS-d4ba4116-dc10-4d04-b2c1-862ced4dee34:NORMAL:127.0.0.1:44434|FINALIZED]]} size 0
[junit4] 2> 624402 INFO (IPC Server handler 1 on 36700) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60774 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-d4ba4116-dc10-4d04-b2c1-862ced4dee34:NORMAL:127.0.0.1:44434|FINALIZED], ReplicaUC[[DISK]DS-9ee29c27-84a0-48dc-88c2-cfeda54ca458:NORMAL:127.0.0.1:60774|FINALIZED]]} size 0
[junit4] 2> 624423 WARN (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 624482 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 624482 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 624482 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 624513 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 624513 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 624522 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8143478495655935]
[junit4] 2> 624590 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@750df1ac[control_collection_shard1_replica_n1] main]
[junit4] 2> 624591 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 624591 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 624591 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 624592 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573633340633776128
[junit4] 2> 624608 INFO (searcherExecutor-2285-thread-1-processing-n:127.0.0.1:41392_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@750df1ac[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 624634 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 624634 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 624634 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41392/control_collection_shard1_replica_n1/
[junit4] 2> 624634 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 624634 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:41392/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 624634 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 624647 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41392/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 624749 INFO (zkCallback-1078-thread-1-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 624798 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 624800 INFO (qtp1666203318-7921) [n:127.0.0.1:41392_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1792
[junit4] 2> 624804 INFO (qtp1666203318-7924) [n:127.0.0.1:41392_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 624896 INFO (OverseerCollectionConfigSetProcessor-98352083599425540-127.0.0.1:41392_-n_0000000000) [n:127.0.0.1:41392_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 624902 INFO (zkCallback-1078-thread-1-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 625804 INFO (qtp1666203318-7924) [n:127.0.0.1:41392_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:41392_&wt=javabin&version=2} status=0 QTime=2928
[junit4] 2> 625919 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 625919 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:47214/solr ready
[junit4] 2> 625919 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 625922 INFO (qtp1666203318-7925) [n:127.0.0.1:41392_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 625939 INFO (OverseerThreadFactory-2282-thread-2-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 625941 WARN (OverseerThreadFactory-2282-thread-2-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 626148 INFO (qtp1666203318-7925) [n:127.0.0.1:41392_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 626148 INFO (qtp1666203318-7925) [n:127.0.0.1:41392_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=226
[junit4] 2> 626385 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-1-001 of type NRT
[junit4] 2> 626385 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 626389 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@491fa91c{/,null,AVAILABLE}
[junit4] 2> 626389 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c4b21a2{HTTP/1.1,[http/1.1]}{127.0.0.1:53183}
[junit4] 2> 626389 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.Server Started @628572ms
[junit4] 2> 626389 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:36700/hdfs__localhost_36700__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=53183, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-1-001/cores}
[junit4] 2> 626390 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 626390 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 626390 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 626390 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 626390 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-22T14:28:22.201Z
[junit4] 2> 626418 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 626418 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-1-001/solr.xml
[junit4] 2> 626422 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 626431 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47214/solr
[junit4] 2> 626447 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 626448 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 626450 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53183_
[junit4] 2> 626450 INFO (zkCallback-1078-thread-1-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 626458 INFO (zkCallback-1085-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 626468 INFO (zkCallback-1090-thread-2-processing-n:127.0.0.1:53183_) [n:127.0.0.1:53183_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 626521 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 626531 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 626531 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 626532 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:53183_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-1-001/cores
[junit4] 2> 626570 INFO (qtp1666203318-7923) [n:127.0.0.1:41392_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:53183_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 626571 INFO (OverseerCollectionConfigSetProcessor-98352083599425540-127.0.0.1:41392_-n_0000000000) [n:127.0.0.1:41392_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 626572 INFO (OverseerThreadFactory-2282-thread-3-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:53183_ for creating new replica
[junit4] 2> 626575 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 626575 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 627588 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 627600 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
[junit4] 2> 627875 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 627915 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
[junit4] 2> 627916 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 627916 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:36700/solr_hdfs_home
[junit4] 2> 627916 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 627916 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 627916 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-1-001/cores/collection1_shard1_replica_n1], dataDir=[hdfs://localhost:36700/solr_hdfs_home/collection1/core_node2/data/]
[junit4] 2> 627917 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
[junit4] 2> 627924 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 627924 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 627924 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 627957 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 627959 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/collection1/core_node2/data
[junit4] 2> 627991 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/collection1/core_node2/data/index
[junit4] 2> 628003 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 628003 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 628003 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 628019 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 628019 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=50, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1339697012392608]
[junit4] 2> 628113 INFO (IPC Server handler 1 on 36700) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44434 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bf1ea655-26f7-49cc-8bae-6ca8f9e0893c:NORMAL:127.0.0.1:60774|RBW], ReplicaUC[[DISK]DS-c134b49c-5a84-4302-bfb3-e2acecd87a90:NORMAL:127.0.0.1:44434|FINALIZED]]} size 0
[junit4] 2> 628113 INFO (IPC Server handler 2 on 36700) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60774 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bf1ea655-26f7-49cc-8bae-6ca8f9e0893c:NORMAL:127.0.0.1:60774|RBW], ReplicaUC[[DISK]DS-c134b49c-5a84-4302-bfb3-e2acecd87a90:NORMAL:127.0.0.1:44434|FINALIZED]]} size 0
[junit4] 2> 628144 WARN (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 628302 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 628302 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 628302 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 628316 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 628316 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 628321 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8143478495655935]
[junit4] 2> 628337 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@20fba71b[collection1_shard1_replica_n1] main]
[junit4] 2> 628338 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 628338 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 628338 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 628339 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573633344562790400
[junit4] 2> 628343 INFO (searcherExecutor-2296-thread-1-processing-n:127.0.0.1:53183_ x:collection1_shard1_replica_n1 s:shard1 c:collection1) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@20fba71b[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 628347 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 628347 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 628347 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53183/collection1_shard1_replica_n1/
[junit4] 2> 628347 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 628347 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:53183/collection1_shard1_replica_n1/ has no replicas
[junit4] 2> 628347 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 628349 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53183/collection1_shard1_replica_n1/ shard1
[junit4] 2> 628500 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 628501 INFO (qtp1655918669-7975) [n:127.0.0.1:53183_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1926
[junit4] 2> 628512 INFO (qtp1666203318-7923) [n:127.0.0.1:41392_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:53183_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1942
[junit4] 2> 628578 INFO (OverseerCollectionConfigSetProcessor-98352083599425540-127.0.0.1:41392_-n_0000000000) [n:127.0.0.1:41392_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 628838 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-2-001 of type NRT
[junit4] 2> 628839 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 628839 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2544ed8c{/,null,AVAILABLE}
[junit4] 2> 628840 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@745bd098{HTTP/1.1,[http/1.1]}{127.0.0.1:50114}
[junit4] 2> 628840 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.e.j.s.Server Started @631022ms
[junit4] 2> 628840 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:36700/hdfs__localhost_36700__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=50114, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-2-001/cores}
[junit4] 2> 628840 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 628841 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 628841 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 628841 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 628858 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-22T14:28:24.652Z
[junit4] 2> 628925 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 628925 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-2-001/solr.xml
[junit4] 2> 628972 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 628986 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47214/solr
[junit4] 2> 629129 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 629132 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 629150 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50114_
[junit4] 2> 629151 INFO (zkCallback-1090-thread-2-processing-n:127.0.0.1:53183_) [n:127.0.0.1:53183_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 629152 INFO (zkCallback-1085-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 629152 INFO (zkCallback-1078-thread-3-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 629152 INFO (zkCallback-1096-thread-1-processing-n:127.0.0.1:50114_) [n:127.0.0.1:50114_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 629279 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 629283 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 629283 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 629284 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[2C9DFF6E29E6CD07]) [n:127.0.0.1:50114_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-2-001/cores
[junit4] 2> 629311 INFO (qtp1666203318-7926) [n:127.0.0.1:41392_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:50114_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 629313 INFO (OverseerThreadFactory-2282-thread-4-processing-n:127.0.0.1:41392_) [n:127.0.0.1:41392_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:50114_ for creating new replica
[junit4] 2> 629321 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n3&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 629321 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 630358 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 630461 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema [collection1_shard1_replica_n3] Schema name=test
[junit4] 2> 630890 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 630949 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n3' using configuration from collection collection1, trusted=true
[junit4] 2> 630952 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n3' (registry 'solr.core.collection1.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@784ebf91
[junit4] 2> 630952 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:36700/solr_hdfs_home
[junit4] 2> 630953 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 630953 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 630953 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.SolrCore [[collection1_shard1_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_2C9DFF6E29E6CD07-001/shard-2-001/cores/collection1_shard1_replica_n3], dataDir=[hdfs://localhost:36700/solr_hdfs_home/collection1/core_node4/data/]
[junit4] 2> 630955 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/collection1/core_node4/data/snapshot_metadata
[junit4] 2> 630975 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 630975 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 630975 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 631000 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 631001 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/collection1/core_node4/data
[junit4] 2> 631117 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36700/solr_hdfs_home/collection1/core_node4/data/index
[junit4] 2> 631168 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 631168 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 631168 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 631193 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 631198 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=50, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1339697012392608]
[junit4] 2> 631349 INFO (IPC Server handler 4 on 36700) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44434 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-9ee29c27-84a0-48dc-88c2-cfeda54ca458:NORMAL:127.0.0.1:60774|RBW], ReplicaUC[[DISK]DS-d4ba4116-dc10-4d04-b2c1-862ced4dee34:NORMAL:127.0.0.1:44434|FINALIZED]]} size 0
[junit4] 2> 631357 INFO (IPC Server handler 5 on 36700) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60774 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-9ee29c27-84a0-48dc-88c2-cfeda54ca458:NORMAL:127.0.0.1:60774|RBW], ReplicaUC[[DISK]DS-d4ba4116-dc10-4d04-b2c1-862ced4dee34:NORMAL:127.0.0.1:44434|FINALIZED]]} size 0
[junit4] 2> 631393 WARN (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 631727 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 631727 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 631727 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 631752 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 631752 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 631763 INFO (qtp413784967-8015) [n:127.0.0.1:50114_ c:collection1 s:shard1 x:collection1_shard1_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1
[...truncated too long message...]
pool BP-1778840234-10.41.0.5-1500735389400 (Datanode Uuid 43ffcbf8-e179-4d3b-978d-32d4f88108fc) service to localhost/127.0.0.1:53305
[junit4] 2> 2362035 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2362139 ERROR (SUITE-SharedFSAutoReplicaFailoverTest-seed#[2C9DFF6E29E6CD07]-worker) [ ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
[junit4] 2> java.lang.reflect.InvocationTargetException
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2C9DFF6E29E6CD07-001
[junit4] 2> Jul 22, 2017 2:57:17 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1684, maxMBSortInHeap=6.991688812027187, sim=RandomSimilarity(queryNorm=true): {}, locale=ms-MY, timezone=Europe/Zagreb
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=2,free=260579976,total=523239424
[junit4] 2> NOTE: All tests run in this JVM: [TestConfig, SolrShardReporterTest, TestRangeQuery, CheckHdfsIndexTest, TestBinaryResponseWriter, TestPHPSerializedResponseWriter, MigrateRouteKeyTest, TestSolrFieldCacheBean, CdcrRequestHandlerTest, TestStressCloudBlindAtomicUpdates, XmlUpdateRequestHandlerTest, TestSerializedLuceneMatchVersion, TestSolrQueryParser, TestBlendedInfixSuggestions, ConjunctionSolrSpellCheckerTest, HdfsRecoveryZkTest, TestReplicationHandler, TestCloudRecovery, TestInPlaceUpdatesStandalone, TestFieldCacheWithThreads, DistributedFacetExistsSmallTest, TestHighlightDedupGrouping, TestDynamicLoading, TestCoreAdminApis, TestLMDirichletSimilarityFactory, TestAuthenticationFramework, TestFieldTypeCollectionResource, DeleteLastCustomShardedReplicaTest, ConnectionReuseTest, HdfsWriteToMultipleCollectionsTest, SolrCoreMetricManagerTest, TestQuerySenderNoQuery, TestCrossCoreJoin, TestExtendedDismaxParser, InfoHandlerTest, BlockJoinFacetDistribTest, CloneFieldUpdateProcessorFactoryTest, TlogReplayBufferedWhileIndexingTest, TestApiFramework, TestManagedResourceStorage, RecoveryAfterSoftCommitTest, FieldMutatingUpdateProcessorTest, TestScoreJoinQPNoScore, SecurityConfHandlerTest, TestDFISimilarityFactory, SolrCLIZkUtilsTest, NodeMutatorTest, TestShardHandlerFactory, SolrPluginUtilsTest, BufferStoreTest, CollectionTooManyReplicasTest, HdfsBasicDistributedZkTest, SyncSliceTest, BasicAuthIntegrationTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestSystemCollAutoCreate, TestDocTermOrds, TestWordDelimiterFilterFactory, TestBadConfig, TestOmitPositions, TestLuceneMatchVersion, JavabinLoaderTest, TestCollationField, TestRequestForwarding, TestSha256AuthenticationProvider, TestOrdValues, CoreSorterTest, TestStressLucene, PeerSyncTest, MBeansHandlerTest, SolrRequestParserTest, TestRemoteStreaming, OutputWriterTest, TestConfigReload, ReplicationFactorTest, BasicDistributedZkTest, HdfsCollectionsAPIDistributedZkTest, TestSchemaResource, UpdateParamsTest, TestMiniSolrCloudClusterSSL, EchoParamsTest, HLLUtilTest, TestGeoJSONResponseWriter, TestComplexPhraseLeadingWildcard, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [696/727 (3!)] on J0 in 58.37s, 1 test, 1 failure <<< FAILURES!
[...truncated 43302 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1359 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1359/
2 tests failed.
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
Error Message:
expected:<[{indexVersion=1500648276317,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.doc, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6.tim, _6.tip, _7.doc, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7.tim, _7.tip, segments_2]}]> but was:<[{indexVersion=1500648276317,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.doc, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6.tim, _6.tip, _7.doc, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7.tim, _7.tip, segments_2]}, {indexVersion=1500648276317,generation=3,filelist=[_8.cfe, _8.cfs, _8.si, segments_3]}]>
Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1500648276317,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.doc, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6.tim, _6.tip, _7.doc, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7.tim, _7.tip, segments_2]}]> but was:<[{indexVersion=1500648276317,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.doc, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6.tim, _6.tip, _7.doc, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7.tim, _7.tip, segments_2]}, {indexVersion=1500648276317,generation=3,filelist=[_8.cfe, _8.cfs, _8.si, segments_3]}]>
at __randomizedtesting.SeedInfo.seed([B638DFBC8C6CA24A:93EFC48CFC24AC49]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
expected:<2> but was:<4>
Stack Trace:
java.lang.AssertionError: expected:<2> but was:<4>
at __randomizedtesting.SeedInfo.seed([B638DFBC8C6CA24A:3E6CE0662290CFB2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11300 lines...]
[junit4] Suite: org.apache.solr.handler.TestReplicationHandler
[junit4] 2> 89101 INFO (SUITE-TestReplicationHandler-seed#[B638DFBC8C6CA24A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/init-core-data-001
[junit4] 2> 89102 WARN (SUITE-TestReplicationHandler-seed#[B638DFBC8C6CA24A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=69 numCloses=69
[junit4] 2> 89102 INFO (SUITE-TestReplicationHandler-seed#[B638DFBC8C6CA24A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 89103 INFO (SUITE-TestReplicationHandler-seed#[B638DFBC8C6CA24A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 89111 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting doTestDetails
[junit4] 2> 89111 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/collection1
[junit4] 2> 89114 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 89119 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@76b59f31{/solr,null,AVAILABLE}
[junit4] 2> 89121 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d54e3{HTTP/1.1,[http/1.1]}{127.0.0.1:34466}
[junit4] 2> 89121 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server Started @91364ms
[junit4] 2> 89121 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=34466}
[junit4] 2> 89122 ERROR (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 89122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 89122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 89122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 89122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-21T14:39:10.547Z
[junit4] 2> 89122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/solr.xml
[junit4] 2> 89127 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 89152 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89155 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89155 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89157 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/.
[junit4] 2> 89158 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 89174 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 89181 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 89182 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 89183 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/./collection1, trusted=true
[junit4] 2> 89183 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89183 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 89183 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/./collection1/data/]
[junit4] 2> 89185 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=38.5947265625, floorSegmentMB=1.4560546875, forceMergeDeletesPctAllowed=18.808908817076464, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.38426369188841947
[junit4] 2> 89217 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 89217 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 89218 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@23768e01
[junit4] 2> 89218 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c436c3d[collection1] main]
[junit4] 2> 89218 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-001/collection1/conf
[junit4] 2> 89219 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
[junit4] 2> 89219 INFO (coreLoadExecutor-411-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 89220 INFO (searcherExecutor-412-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7c436c3d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 89681 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1
[junit4] 2> 89684 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 89685 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18501237{/solr,null,AVAILABLE}
[junit4] 2> 89685 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4781d4d1{HTTP/1.1,[http/1.1]}{127.0.0.1:50677}
[junit4] 2> 89685 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server Started @91928ms
[junit4] 2> 89685 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=50677}
[junit4] 2> 89686 ERROR (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 89686 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 89686 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 89686 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 89686 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-21T14:39:11.111Z
[junit4] 2> 89686 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/solr.xml
[junit4] 2> 89691 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 89721 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89727 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89727 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89730 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/.
[junit4] 2> 89730 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 89740 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 89747 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 89749 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 89749 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/./collection1, trusted=true
[junit4] 2> 89750 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 89750 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 89750 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/./collection1/data/]
[junit4] 2> 89752 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=38.5947265625, floorSegmentMB=1.4560546875, forceMergeDeletesPctAllowed=18.808908817076464, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.38426369188841947
[junit4] 2> 89801 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 89801 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 89802 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@41254d79
[junit4] 2> 89803 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1a1fdb9a[collection1] main]
[junit4] 2> 89803 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1/conf
[junit4] 2> 89807 INFO (coreLoadExecutor-421-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
[junit4] 2> 89808 INFO (searcherExecutor-422-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1a1fdb9a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 90042 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 90043 INFO (indexFetcher-426-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 90043 INFO (indexFetcher-426-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 90043 INFO (indexFetcher-426-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 90043 INFO (indexFetcher-426-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 90272 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4781d4d1{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 90272 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2012333697
[junit4] 2> 90273 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 90273 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 90273 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 90274 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 90274 INFO (coreCloseExecutor-427-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@7b89a099
[junit4] 2> 90277 INFO (coreCloseExecutor-427-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=2072617113
[junit4] 2> 90279 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@18501237{/solr,null,UNAVAILABLE}
[junit4] 2> 90281 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 90282 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63529aea{/solr,null,AVAILABLE}
[junit4] 2> 90282 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5cbefe34{HTTP/1.1,[http/1.1]}{127.0.0.1:45687}
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server Started @92526ms
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=45687}
[junit4] 2> 90283 ERROR (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-21T14:39:11.708Z
[junit4] 2> 90283 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/solr.xml
[junit4] 2> 90288 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 90326 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 90333 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 90333 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 90335 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/.
[junit4] 2> 90336 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 90346 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 90366 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 90367 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 90367 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/./collection1, trusted=true
[junit4] 2> 90368 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 90368 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 90368 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/./collection1/data/]
[junit4] 2> 90370 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=38.5947265625, floorSegmentMB=1.4560546875, forceMergeDeletesPctAllowed=18.808908817076464, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.38426369188841947
[junit4] 2> 90407 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 90407 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 90408 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3cbf4890
[junit4] 2> 90409 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7e88b015[collection1] main]
[junit4] 2> 90409 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-002/collection1/conf
[junit4] 2> 90410 INFO (coreLoadExecutor-434-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
[junit4] 2> 90411 INFO (searcherExecutor-435-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7e88b015[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 90413 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 90414 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 90414 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 90414 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 90414 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 90868 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 90871 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=1
[junit4] 2> 90875 INFO (qtp1710725223-2597) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={slave=false&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 90875 INFO (qtp1409625958-2639) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=2
[junit4] 2> 90878 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 1
[junit4] 2> 90879 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 90879 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b3fb4aa commitCommandVersion:0
[junit4] 2> 90882 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4672c909[collection1] main]
[junit4] 2> 90883 INFO (searcherExecutor-412-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4672c909[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1)))}
[junit4] 2> 90883 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 90883 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 4
[junit4] 2> 90887 WARN (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
[junit4] 2> 90889 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 90889 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 2
[junit4] 2> 90889 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 1500647952304
[junit4] 2> 90889 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 90889 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 90889 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 90942 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=51
[junit4] 2> 90942 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Number of files in latest index in master: 10
[junit4] 2> 90946 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@f385b01
[junit4] 2> 90946 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 90997 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@46c08c42 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b777b75)
[junit4] 2> 90999 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91001 INFO (qtp1710725223-2597) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91003 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91005 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91007 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91008 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91010 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91011 INFO (qtp1710725223-2597) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91013 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91014 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91015 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1480) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@46c08c42 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b777b75)
[junit4] 2> 91018 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=152917920, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 91019 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 91020 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@350592e5[collection1] main]
[junit4] 2> 91021 INFO (searcherExecutor-435-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@350592e5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1)))}
[junit4] 2> 91021 INFO (qtp1409625958-2641) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34466/solr/collection1/replication&command=fetchindex} status=0 QTime=134
[junit4] 2> 91026 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={slave=false&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 91027 INFO (qtp1409625958-2640) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=3
[junit4] 2> 91028 INFO (qtp1409625958-2643) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=1 status=0 QTime=0
[junit4] 2> 91029 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 1 docs
[junit4] 2> 91031 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 1
[junit4] 2> 91032 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 91032 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b3fb4aa commitCommandVersion:0
[junit4] 2> 91035 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@20759dbc[collection1] main]
[junit4] 2> 91035 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 91036 INFO (searcherExecutor-412-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@20759dbc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1) Uninverting(_1(8.0.0):C1)))}
[junit4] 2> 91036 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 4
[junit4] 2> 91037 WARN (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
[junit4] 2> 91039 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 91040 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 3
[junit4] 2> 91040 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 1500647952457
[junit4] 2> 91040 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 91040 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 1500647952304
[junit4] 2> 91040 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 91041 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=0
[junit4] 2> 91042 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Number of files in latest index in master: 19
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4c9d5f88
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@241f8cd3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7de29a7d)
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.si because it already exists
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.nvd because it already exists
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.tip because it already exists
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.fdx because it already exists
[junit4] 2> 91044 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.doc because it already exists
[junit4] 2> 91045 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.tim because it already exists
[junit4] 2> 91045 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.fdt because it already exists
[junit4] 2> 91045 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.fnm because it already exists
[junit4] 2> 91045 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.nvm because it already exists
[junit4] 2> 91046 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91047 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91049 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91050 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91052 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91054 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91055 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91056 INFO (qtp1710725223-2604) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91057 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=_1.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91059 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=3&qt=/replication&file=segments_3&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91059 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1545) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@241f8cd3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7de29a7d)
[junit4] 2> 91062 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=152917920, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 91063 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 91063 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@751d8d7e[collection1] main]
[junit4] 2> 91065 INFO (searcherExecutor-435-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@751d8d7e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1) Uninverting(_1(8.0.0):C1)))}
[junit4] 2> 91065 INFO (qtp1409625958-2642) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34466/solr/collection1/replication&command=fetchindex} status=0 QTime=27
[junit4] 2> 91067 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={slave=false&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 91068 INFO (qtp1409625958-2636) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=1
[junit4] 2> 91069 INFO (qtp1409625958-2637) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=2 status=0 QTime=0
[junit4] 2> 91070 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 2 docs
[junit4] 2> 91071 INFO (qtp1710725223-2597) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
[junit4] 2> 91072 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 91072 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b3fb4aa commitCommandVersion:0
[junit4] 2> 91079 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@148b4b9e[collection1] main]
[junit4] 2> 91080 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 91081 INFO (searcherExecutor-412-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@148b4b9e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1) Uninverting(_1(8.0.0):C1) Uninverting(_2(8.0.0):C1)))}
[junit4] 2> 91082 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 10
[junit4] 2> 91083 WARN (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
[junit4] 2> 91085 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 91086 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 4
[junit4] 2> 91086 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 1500647952497
[junit4] 2> 91086 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 3
[junit4] 2> 91086 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 1500647952457
[junit4] 2> 91086 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 91087 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=0
[junit4] 2> 91088 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Number of files in latest index in master: 28
[junit4] 2> 91090 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3c5c800b
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@7cbe2705 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b18bdd6)
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.si because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.nvd because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.tip because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.fdx because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.doc because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.tim because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.fdt because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.fnm because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _0.nvm because it already exists
[junit4] 2> 91091 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.si because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.nvd because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.doc because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.tim because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.tip because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.fdx because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.fnm because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.nvm because it already exists
[junit4] 2> 91092 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Skipping download for _1.fdt because it already exists
[junit4] 2> 91093 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91094 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91096 INFO (qtp1710725223-2597) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91097 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91099 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91101 INFO (qtp1710725223-2598) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91103 INFO (qtp1710725223-2601) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91104 INFO (qtp1710725223-2602) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91105 INFO (qtp1710725223-2597) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=_2.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91107 INFO (qtp1710725223-2603) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=4&qt=/replication&file=segments_4&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 91107 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1610) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@7cbe2705 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b18bdd6)
[junit4] 2> 91109 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=152917920, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 91110 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 91112 INFO (explicit-fetchindex-cmd) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3919310e[collection1] main]
[junit4] 2> 91113 INFO (searcherExecutor-435-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3919310e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1) Uninverting(_1(8.0.0):C1) Uninverting(_2(8.0.0):C1)))}
[junit4] 2> 91113 INFO (qtp1409625958-2639) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:34466/solr/collection1/replication&command=fetchindex} status=0 QTime=29
[junit4] 2> 91114 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/collection1
[junit4] 2> 91117 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 91120 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e729d83{/solr,null,AVAILABLE}
[junit4] 2> 91121 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2f9ae5e3{HTTP/1.1,[http/1.1]}{127.0.0.1:48788}
[junit4] 2> 91121 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server Started @93364ms
[junit4] 2> 91121 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/collection1/data, hostContext=/solr, hostPort=48788}
[junit4] 2> 91122 ERROR (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 91122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 91122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 91122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 91122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-21T14:39:12.547Z
[junit4] 2> 91122 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/solr.xml
[junit4] 2> 91127 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 91152 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 91157 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 91157 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 91159 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/.
[junit4] 2> 91159 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 91179 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 91197 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 91199 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 91199 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/./collection1, trusted=true
[junit4] 2> 91199 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214
[junit4] 2> 91199 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 91199 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/./collection1/data/]
[junit4] 2> 91202 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=38.5947265625, floorSegmentMB=1.4560546875, forceMergeDeletesPctAllowed=18.808908817076464, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.38426369188841947
[junit4] 2> 91242 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 91242 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 91243 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@36898296
[junit4] 2> 91244 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@603fda17[collection1] main]
[junit4] 2> 91244 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-003/collection1/conf
[junit4] 2> 91245 WARN (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
[junit4] 2> 91245 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler No value set for 'pollInterval'. Timer Task not started.
[junit4] 2> 91246 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml
[junit4] 2> 91246 INFO (coreLoadExecutor-448-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 91247 INFO (searcherExecutor-449-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@603fda17[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 91412 INFO (qtp1710725223-2600) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 91413 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 4
[junit4] 2> 91413 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 1500647952497
[junit4] 2> 91413 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 4
[junit4] 2> 91413 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 1500647952497
[junit4] 2> 91413 INFO (indexFetcher-439-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave in sync with master.
[junit4] 2> 91706 INFO (qtp1710725223-2601) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 91711 WARN (qtp1591868729-2667) [ x:collection1] o.a.s.h.ReplicationHandler Exception while invoking 'details' method for replication on master
[junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34466/solr: Expected mime type application/octet-stream but got text/html. <html>
[junit4] 2> <head>
[junit4] 2> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
[junit4] 2> <title>Error 404 </title>
[junit4] 2> </head>
[junit4] 2> <body>
[junit4] 2> <h2>HTTP ERROR: 404</h2>
[junit4] 2> <p>Problem accessing /solr/replication. Reason:
[junit4] 2> <pre> Can not find: /solr/replication</pre></p>
[junit4] 2> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
[junit4] 2> </body>
[junit4] 2> </html>
[junit4] 2>
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:588)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.handler.IndexFetcher.getDetails(IndexFetcher.java:1813)
[junit4] 2> at org.apache.solr.handler.ReplicationHandler.getReplicationDetails(ReplicationHandler.java:952)
[junit4] 2> at org.apache.solr.handler.ReplicationHandler.handleRequestBody(ReplicationHandler.java:330)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:378)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:322)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 91712 INFO (qtp1591868729-2667) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=8
[junit4] 2> 91713 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2f9ae5e3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 91713 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=137784583
[junit4] 2> 91713 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 91714 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 91714 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 91714 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 91715 INFO (coreCloseExecutor-453-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@40c90fd0
[junit4] 2> 91717 INFO (coreCloseExecutor-453-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1086918608
[junit4] 2> 91719 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2e729d83{/solr,null,UNAVAILABLE}
[junit4] 2> 91719 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending doTestDetails
[junit4] 2> 91720 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5d54e3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 91721 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1076225162
[junit4] 2> 91721 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 91723 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 91725 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 91726 INFO (coreCloseExecutor-455-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@180ef780
[junit4] 2> 91729 INFO (coreCloseExecutor-455-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=403634048
[junit4] 2> 91732 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@76b59f31{/solr,null,UNAVAILABLE}
[junit4] 2> 91733 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5cbefe34{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 91734 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=755003924
[junit4] 2> 91734 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 91734 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 91734 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 91734 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 91734 INFO (coreCloseExecutor-457-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@6ffa9078
[junit4] 2> 91736 INFO (coreCloseExecutor-457-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1878691960
[junit4] 2> 91744 INFO (TEST-TestReplicationHandler.doTestDetails-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@63529aea{/solr,null,UNAVAILABLE}
[junit4] 2> 91748 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting doTestIllegalFilePaths
[junit4] 2> 91749 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-004/collection1
[junit4] 2> 91752 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 91756 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c66454e{/solr,null,AVAILABLE}
[junit4] 2> 91757 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6170d23b{HTTP/1.1,[http/1.1]}{127.0.0.1:45008}
[junit4] 2> 91757 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.e.j.s.Server Started @94000ms
[junit4] 2> 91757 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-004/collection1/data, hostContext=/solr, hostPort=45008}
[junit4] 2> 91760 ERROR (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 91760 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 91761 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 91761 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 91761 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-21T14:39:13.186Z
[junit4] 2> 91761 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_B638DFBC8C6CA24A-001/solr-instance-004/solr.xml
[junit4] 2> 91765 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5efd0214, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 91789 INFO (TEST-TestReplicationHandler.doTestIllegalFilePaths-seed#[B638DFBC8C6CA24A]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled
[...truncated too long message...]
lConnectorWithSafeStartup@localhost:47916) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> 691787 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_B638DFBC8C6CA24A-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_B638DFBC8C6CA24A-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:55220) [ ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-1911394687-10.41.0.5-1500648459753 (Datanode Uuid f5b17416-a4be-45c1-aa1a-58887608709a) service to localhost/127.0.0.1:55220 interrupted
[junit4] 2> 691787 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_B638DFBC8C6CA24A-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_B638DFBC8C6CA24A-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:55220) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1911394687-10.41.0.5-1500648459753 (Datanode Uuid f5b17416-a4be-45c1-aa1a-58887608709a) service to localhost/127.0.0.1:55220
[junit4] 2> 691818 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[B638DFBC8C6CA24A]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 691920 ERROR (SUITE-SharedFSAutoReplicaFailoverTest-seed#[B638DFBC8C6CA24A]-worker) [ ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
[junit4] 2> java.lang.reflect.InvocationTargetException
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_B638DFBC8C6CA24A-001
[junit4] 2> Jul 21, 2017 2:49:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=7501, maxDocsPerChunk=634, blockSize=767), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=7501, blockSize=767)), sim=RandomSimilarity(queryNorm=false): {}, locale=th, timezone=Antarctica/Rothera
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=2,free=250632208,total=513802240
[junit4] 2> NOTE: All tests run in this JVM: [TestLRUStatsCache, SpellCheckCollatorTest, RecoveryAfterSoftCommitTest, TestNestedDocsSort, TestRecoveryHdfs, TestReplicationHandler, TestMultiValuedNumericRangeQuery, TestManagedResource, TestDistribIDF, TestSolrConfigHandlerConcurrent, LeaderElectionIntegrationTest, TestRuleBasedAuthorizationPlugin, ReplicationFactorTest, RAMDirectoryFactoryTest, TestTolerantUpdateProcessorRandomCloud, TestSchemaVersionResource, UpdateParamsTest, TestFieldCacheVsDocValues, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [108/728 (2!)] on J0 in 102.84s, 1 test, 1 failure <<< FAILURES!
[...truncated 45269 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1358 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1358/
3 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Error from server at http://127.0.0.1:40457/aip/w/control_collection: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /aip/w/control_collection/update. Reason: <pre> Can not find: /aip/w/control_collection/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40457/aip/w/control_collection: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /aip/w/control_collection/update. Reason:
<pre> Can not find: /aip/w/control_collection/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:588)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1580)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:656)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
Error Message:
expected:<[{indexVersion=1500564090624,generation=2,filelist=[_7k.cfe, _7k.cfs, _7k.si, _7o.fdt, _7o.fdx, _7o.fnm, _7o.nvd, _7o.nvm, _7o.si, _7o_Memory_0.ram, _7s.cfe, _7s.cfs, _7s.si, _7t.fdt, _7t.fdx, _7t.fnm, _7t.nvd, _7t.nvm, _7t.si, _7t_Memory_0.ram, _7x.fdt, _7x.fdx, _7x.fnm, _7x.nvd, _7x.nvm, _7x.si, _7x_Memory_0.ram, _80.cfe, _80.cfs, _80.si, _81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Memory_0.ram, _82.fdt, _82.fdx, _82.fnm, _82.nvd, _82.nvm, _82.si, _82_Memory_0.ram, _84.fdt, _84.fdx, _84.fnm, _84.nvd, _84.nvm, _84.si, _84_Memory_0.ram, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Memory_0.ram, _87.fdt, _87.fdx, _87.fnm, _87.nvd, _87.nvm, _87.si, _87_Memory_0.ram, _89.fdt, _89.fdx, _89.fnm, _89.nvd, _89.nvm, _89.si, _89_Memory_0.ram, _8a.cfe, _8a.cfs, _8a.si, _8b.fdt, _8b.fdx, _8b.fnm, _8b.nvd, _8b.nvm, _8b.si, _8b_Memory_0.ram, segments_2]}]> but was:<[{indexVersion=1500564090624,generation=2,filelist=[_7k.cfe, _7k.cfs, _7k.si, _7o.fdt, _7o.fdx, _7o.fnm, _7o.nvd, _7o.nvm, _7o.si, _7o_Memory_0.ram, _7s.cfe, _7s.cfs, _7s.si, _7t.fdt, _7t.fdx, _7t.fnm, _7t.nvd, _7t.nvm, _7t.si, _7t_Memory_0.ram, _7x.fdt, _7x.fdx, _7x.fnm, _7x.nvd, _7x.nvm, _7x.si, _7x_Memory_0.ram, _80.cfe, _80.cfs, _80.si, _81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Memory_0.ram, _82.fdt, _82.fdx, _82.fnm, _82.nvd, _82.nvm, _82.si, _82_Memory_0.ram, _84.fdt, _84.fdx, _84.fnm, _84.nvd, _84.nvm, _84.si, _84_Memory_0.ram, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Memory_0.ram, _87.fdt, _87.fdx, _87.fnm, _87.nvd, _87.nvm, _87.si, _87_Memory_0.ram, _89.fdt, _89.fdx, _89.fnm, _89.nvd, _89.nvm, _89.si, _89_Memory_0.ram, _8a.cfe, _8a.cfs, _8a.si, _8b.fdt, _8b.fdx, _8b.fnm, _8b.nvd, _8b.nvm, _8b.si, _8b_Memory_0.ram, segments_2]}, {indexVersion=1500564090624,generation=3,filelist=[_7t.fdt, _7t.fdx, _7t.fnm, _7t.nvd, _7t.nvm, _7t.si, _7t_Memory_0.ram, _8c.fdt, _8c.fdx, _8c.fnm, _8c.nvd, _8c.nvm, _8c.si, _8c_Memory_0.ram, segments_3]}]>
Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1500564090624,generation=2,filelist=[_7k.cfe, _7k.cfs, _7k.si, _7o.fdt, _7o.fdx, _7o.fnm, _7o.nvd, _7o.nvm, _7o.si, _7o_Memory_0.ram, _7s.cfe, _7s.cfs, _7s.si, _7t.fdt, _7t.fdx, _7t.fnm, _7t.nvd, _7t.nvm, _7t.si, _7t_Memory_0.ram, _7x.fdt, _7x.fdx, _7x.fnm, _7x.nvd, _7x.nvm, _7x.si, _7x_Memory_0.ram, _80.cfe, _80.cfs, _80.si, _81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Memory_0.ram, _82.fdt, _82.fdx, _82.fnm, _82.nvd, _82.nvm, _82.si, _82_Memory_0.ram, _84.fdt, _84.fdx, _84.fnm, _84.nvd, _84.nvm, _84.si, _84_Memory_0.ram, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Memory_0.ram, _87.fdt, _87.fdx, _87.fnm, _87.nvd, _87.nvm, _87.si, _87_Memory_0.ram, _89.fdt, _89.fdx, _89.fnm, _89.nvd, _89.nvm, _89.si, _89_Memory_0.ram, _8a.cfe, _8a.cfs, _8a.si, _8b.fdt, _8b.fdx, _8b.fnm, _8b.nvd, _8b.nvm, _8b.si, _8b_Memory_0.ram, segments_2]}]> but was:<[{indexVersion=1500564090624,generation=2,filelist=[_7k.cfe, _7k.cfs, _7k.si, _7o.fdt, _7o.fdx, _7o.fnm, _7o.nvd, _7o.nvm, _7o.si, _7o_Memory_0.ram, _7s.cfe, _7s.cfs, _7s.si, _7t.fdt, _7t.fdx, _7t.fnm, _7t.nvd, _7t.nvm, _7t.si, _7t_Memory_0.ram, _7x.fdt, _7x.fdx, _7x.fnm, _7x.nvd, _7x.nvm, _7x.si, _7x_Memory_0.ram, _80.cfe, _80.cfs, _80.si, _81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Memory_0.ram, _82.fdt, _82.fdx, _82.fnm, _82.nvd, _82.nvm, _82.si, _82_Memory_0.ram, _84.fdt, _84.fdx, _84.fnm, _84.nvd, _84.nvm, _84.si, _84_Memory_0.ram, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Memory_0.ram, _87.fdt, _87.fdx, _87.fnm, _87.nvd, _87.nvm, _87.si, _87_Memory_0.ram, _89.fdt, _89.fdx, _89.fnm, _89.nvd, _89.nvm, _89.si, _89_Memory_0.ram, _8a.cfe, _8a.cfs, _8a.si, _8b.fdt, _8b.fdx, _8b.fnm, _8b.nvd, _8b.nvm, _8b.si, _8b_Memory_0.ram, segments_2]}, {indexVersion=1500564090624,generation=3,filelist=[_7t.fdt, _7t.fdx, _7t.fnm, _7t.nvd, _7t.nvm, _7t.si, _7t_Memory_0.ram, _8c.fdt, _8c.fdx, _8c.fnm, _8c.nvd, _8c.nvm, _8c.si, _8c_Memory_0.ram, segments_3]}]>
at __randomizedtesting.SeedInfo.seed([2CE1C8A43A164520:936D3944A5E4B23]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([2CE1C8A43A164520:A4B5F77E94EA28D8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12069 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/init-core-data-001
[junit4] 2> 1540469 WARN (SUITE-FullSolrCloudDistribCmdsTest-seed#[2CE1C8A43A164520]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=11 numCloses=11
[junit4] 2> 1540469 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[2CE1C8A43A164520]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1540470 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[2CE1C8A43A164520]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 1540470 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[2CE1C8A43A164520]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /aip/w
[junit4] 2> 1540471 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1540472 INFO (Thread-8995) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1540472 INFO (Thread-8995) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1540473 ERROR (Thread-8995) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1540572 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ZkTestServer start zk server on port:45078
[junit4] 2> 1540627 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1540628 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1540629 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1540630 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1540630 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1540631 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1540632 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1540632 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1540633 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1540633 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1540634 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1540635 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1540725 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1540725 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@38fd9b38{/aip/w,null,AVAILABLE}
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@43b4b9ff{HTTP/1.1,[http/1.1]}{127.0.0.1:40457}
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server Started @1542575ms
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/tempDir-001/control/data, hostContext=/aip/w, hostPort=40457, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/control-001/cores}
[junit4] 2> 1540726 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1540726 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-20T15:12:08.672Z
[junit4] 2> 1540740 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1540740 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/control-001/solr.xml
[junit4] 2> 1540744 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1540754 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45078/solr
[junit4] 2> 1540925 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1540926 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40457_aip%2Fw
[junit4] 2> 1540927 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.Overseer Overseer (id=98340931398402052-127.0.0.1:40457_aip%2Fw-n_0000000000) starting
[junit4] 2> 1540941 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40457_aip%2Fw
[junit4] 2> 1540948 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1541077 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1541081 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1541081 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1541082 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/control-001/cores
[junit4] 2> 1541100 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1541101 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45078/solr ready
[junit4] 2> 1541103 INFO (qtp1246913222-20289) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:40457_aip%252Fw&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1541105 INFO (OverseerThreadFactory-5363-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1541220 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1541221 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1541327 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1542244 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1542262 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1542364 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1542381 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1542382 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1542382 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1542382 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1542384 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@365b87d2
[junit4] 2> 1542386 WARN (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1542424 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1542424 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1542425 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1542425 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1542426 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=5, maxMergedSegmentMB=66.4169921875, floorSegmentMB=1.376953125, forceMergeDeletesPctAllowed=21.50365019848536, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7268961545949174
[junit4] 2> 1542426 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bda6b02[control_collection_shard1_replica_n1] main]
[junit4] 2> 1542427 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1542427 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1542428 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1542428 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573454904425447424
[junit4] 2> 1542431 INFO (searcherExecutor-5366-thread-1-processing-n:127.0.0.1:40457_aip%2Fw x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@bda6b02[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1542450 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1542450 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1542450 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40457/aip/w/control_collection_shard1_replica_n1/
[junit4] 2> 1542450 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1542450 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:40457/aip/w/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1542450 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1542453 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40457/aip/w/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1542555 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1542603 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1542605 INFO (qtp1246913222-20287) [n:127.0.0.1:40457_aip%2Fw c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1384
[junit4] 2> 1542607 INFO (qtp1246913222-20289) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1542708 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1543108 INFO (OverseerCollectionConfigSetProcessor-98340931398402052-127.0.0.1:40457_aip%2Fw-n_0000000000) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1543608 INFO (qtp1246913222-20289) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:40457_aip%252Fw&wt=javabin&version=2} status=0 QTime=2504
[junit4] 2> 1543622 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1543622 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45078/solr ready
[junit4] 2> 1543622 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1543623 INFO (qtp1246913222-20286) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1543628 INFO (OverseerThreadFactory-5363-thread-2-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1543628 WARN (OverseerThreadFactory-5363-thread-2-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1543838 INFO (qtp1246913222-20286) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1543838 INFO (qtp1246913222-20286) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=215
[junit4] 2> 1543932 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-1-001 of type NRT
[junit4] 2> 1543932 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1543934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@128b89a7{/aip/w,null,AVAILABLE}
[junit4] 2> 1543934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1b92da96{HTTP/1.1,[http/1.1]}{127.0.0.1:50153}
[junit4] 2> 1543934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server Started @1545783ms
[junit4] 2> 1543934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/aip/w, hostPort=50153, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-1-001/cores}
[junit4] 2> 1543934 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1543934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1543934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1543935 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1543935 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-20T15:12:11.881Z
[junit4] 2> 1543953 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1543953 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-1-001/solr.xml
[junit4] 2> 1543958 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1543963 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45078/solr
[junit4] 2> 1543990 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1543992 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1543994 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50153_aip%2Fw
[junit4] 2> 1543996 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1543996 INFO (zkCallback-2580-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1544013 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1544103 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1544108 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1544108 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1544109 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-1-001/cores
[junit4] 2> 1544131 INFO (qtp1246913222-20291) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:50153_aip%252Fw&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1544139 INFO (OverseerCollectionConfigSetProcessor-98340931398402052-127.0.0.1:40457_aip%2Fw-n_0000000000) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1544142 INFO (OverseerThreadFactory-5363-thread-3-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:50153_aip%2Fw for creating new replica
[junit4] 2> 1544159 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1544160 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1544267 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1545180 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1545198 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=test
[junit4] 2> 1545379 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1545397 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using configuration from collection collection1, trusted=true
[junit4] 2> 1545397 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n1' (registry 'solr.core.collection1.shard2.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1545397 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1545397 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-1-001/cores/collection1_shard2_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-1-001/cores/collection1_shard2_replica_n1/data/]
[junit4] 2> 1545400 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@553aef91
[junit4] 2> 1545405 WARN (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1545440 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1545440 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1545441 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1545441 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1545442 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=5, maxMergedSegmentMB=66.4169921875, floorSegmentMB=1.376953125, forceMergeDeletesPctAllowed=21.50365019848536, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7268961545949174
[junit4] 2> 1545442 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b7a573b[collection1_shard2_replica_n1] main]
[junit4] 2> 1545443 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1545443 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1545444 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1545444 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573454907587952640
[junit4] 2> 1545448 INFO (searcherExecutor-5377-thread-1-processing-n:127.0.0.1:50153_aip%2Fw x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [collection1_shard2_replica_n1] Registered new searcher Searcher@4b7a573b[collection1_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1545449 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1545449 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1545449 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50153/aip/w/collection1_shard2_replica_n1/
[junit4] 2> 1545449 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1545449 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:50153/aip/w/collection1_shard2_replica_n1/ has no replicas
[junit4] 2> 1545449 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1545451 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50153/aip/w/collection1_shard2_replica_n1/ shard2
[junit4] 2> 1545553 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1545602 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1545603 INFO (qtp1900711657-20330) [n:127.0.0.1:50153_aip%2Fw c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1443
[junit4] 2> 1545605 INFO (qtp1246913222-20291) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50153_aip%252Fw&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1474
[junit4] 2> 1545705 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1545749 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-2-001 of type NRT
[junit4] 2> 1545775 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1545777 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6fba926c{/aip/w,null,AVAILABLE}
[junit4] 2> 1545777 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@363a9dfa{HTTP/1.1,[http/1.1]}{127.0.0.1:53656}
[junit4] 2> 1545777 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server Started @1547626ms
[junit4] 2> 1545777 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/aip/w, hostPort=53656, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-2-001/cores}
[junit4] 2> 1545778 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1545778 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1545778 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1545778 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1545778 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-20T15:12:13.724Z
[junit4] 2> 1545787 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1545787 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-2-001/solr.xml
[junit4] 2> 1545792 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1545795 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45078/solr
[junit4] 2> 1545810 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1545811 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1545819 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53656_aip%2Fw
[junit4] 2> 1545820 INFO (zkCallback-2580-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1545822 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1545822 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1545827 INFO (zkCallback-2591-thread-1-processing-n:127.0.0.1:53656_aip%2Fw) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1546027 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1546033 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1546044 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1546045 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-2-001/cores
[junit4] 2> 1546095 INFO (qtp1109719522-20360) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:53656_aip%252Fw&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1546097 INFO (OverseerCollectionConfigSetProcessor-98340931398402052-127.0.0.1:40457_aip%2Fw-n_0000000000) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1546098 INFO (OverseerThreadFactory-5363-thread-4-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:53656_aip%2Fw for creating new replica
[junit4] 2> 1546100 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n3&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1546100 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1546203 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1546203 INFO (zkCallback-2591-thread-1-processing-n:127.0.0.1:53656_aip%2Fw) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1547121 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1547138 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.s.IndexSchema [collection1_shard3_replica_n3] Schema name=test
[junit4] 2> 1547287 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1547303 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n3' using configuration from collection collection1, trusted=true
[junit4] 2> 1547304 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n3' (registry 'solr.core.collection1.shard3.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1547304 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1547304 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SolrCore [[collection1_shard3_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-2-001/cores/collection1_shard3_replica_n3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-2-001/cores/collection1_shard3_replica_n3/data/]
[junit4] 2> 1547308 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@27da0334
[junit4] 2> 1547310 WARN (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1547495 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1547495 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1547496 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1547496 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1547497 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=5, maxMergedSegmentMB=66.4169921875, floorSegmentMB=1.376953125, forceMergeDeletesPctAllowed=21.50365019848536, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7268961545949174
[junit4] 2> 1547498 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f3d887f[collection1_shard3_replica_n3] main]
[junit4] 2> 1547498 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1547499 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1547499 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1547500 INFO (searcherExecutor-5388-thread-1-processing-n:127.0.0.1:53656_aip%2Fw x:collection1_shard3_replica_n3 s:shard3 c:collection1) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SolrCore [collection1_shard3_replica_n3] Registered new searcher Searcher@7f3d887f[collection1_shard3_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1547500 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573454909743824896
[junit4] 2> 1547505 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1547505 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1547506 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53656/aip/w/collection1_shard3_replica_n3/
[junit4] 2> 1547506 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1547506 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.SyncStrategy http://127.0.0.1:53656/aip/w/collection1_shard3_replica_n3/ has no replicas
[junit4] 2> 1547506 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1547508 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53656/aip/w/collection1_shard3_replica_n3/ shard3
[junit4] 2> 1547613 INFO (zkCallback-2591-thread-1-processing-n:127.0.0.1:53656_aip%2Fw) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1547613 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1547659 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1547661 INFO (qtp1109719522-20358) [n:127.0.0.1:53656_aip%2Fw c:collection1 s:shard3 x:collection1_shard3_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n3&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1561
[junit4] 2> 1547662 INFO (qtp1109719522-20360) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:53656_aip%252Fw&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=1567
[junit4] 2> 1547664 INFO (OverseerCollectionConfigSetProcessor-98340931398402052-127.0.0.1:40457_aip%2Fw-n_0000000000) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1547759 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-3-001 of type NRT
[junit4] 2> 1547759 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1547760 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@229bea78{/aip/w,null,AVAILABLE}
[junit4] 2> 1547760 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1809488e{HTTP/1.1,[http/1.1]}{127.0.0.1:38592}
[junit4] 2> 1547760 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.e.j.s.Server Started @1549609ms
[junit4] 2> 1547760 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/aip/w, hostPort=38592, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-3-001/cores}
[junit4] 2> 1547761 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1547761 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 1547761 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1547761 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1547761 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-20T15:12:15.707Z
[junit4] 2> 1547762 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1547762 INFO (zkCallback-2591-thread-1-processing-n:127.0.0.1:53656_aip%2Fw) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1547775 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1547775 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-3-001/solr.xml
[junit4] 2> 1547780 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1547782 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45078/solr
[junit4] 2> 1547806 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1547811 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1547814 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38592_aip%2Fw
[junit4] 2> 1547815 INFO (zkCallback-2580-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1547815 INFO (zkCallback-2573-thread-1-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1547815 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1547815 INFO (zkCallback-2591-thread-1-processing-n:127.0.0.1:53656_aip%2Fw) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1547819 INFO (zkCallback-2597-thread-1-processing-n:127.0.0.1:38592_aip%2Fw) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1547929 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1547934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1547934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1547935 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[2CE1C8A43A164520]) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-3-001/cores
[junit4] 2> 1547978 INFO (qtp29921363-20389) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38592_aip%252Fw&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1547981 INFO (OverseerThreadFactory-5363-thread-5-processing-n:127.0.0.1:40457_aip%2Fw) [n:127.0.0.1:40457_aip%2Fw ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:38592_aip%2Fw for creating new replica
[junit4] 2> 1547983 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n5&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1547983 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1548091 INFO (zkCallback-2591-thread-1-processing-n:127.0.0.1:53656_aip%2Fw) [n:127.0.0.1:53656_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1548091 INFO (zkCallback-2597-thread-1-processing-n:127.0.0.1:38592_aip%2Fw) [n:127.0.0.1:38592_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1548099 INFO (zkCallback-2585-thread-1-processing-n:127.0.0.1:50153_aip%2Fw) [n:127.0.0.1:50153_aip%2Fw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1549005 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1549031 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.s.IndexSchema [collection1_shard1_replica_n5] Schema name=test
[junit4] 2> 1549118 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1549172 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n5' using configuration from collection collection1, trusted=true
[junit4] 2> 1549185 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n5' (registry 'solr.core.collection1.shard1.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@185cc62a
[junit4] 2> 1549185 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1549185 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.c.SolrCore [[collection1_shard1_replica_n5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-3-001/cores/collection1_shard1_replica_n5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_2CE1C8A43A164520-001/shard-3-001/cores/collection1_shard1_replica_n5/data/]
[junit4] 2> 1549188 INFO (qtp29921363-20387) [n:127.0.0.1:38592_aip%2Fw c:collection1 s:shard1 x:collection1_shard1_replica_n5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lu
[...truncated too long message...]
430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_2CE1C8A43A164520-001
[junit4] 2> Jul 20, 2017 3:39:30 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=31378, maxDocsPerChunk=679, blockSize=981), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=31378, blockSize=981)), sim=RandomSimilarity(queryNorm=true): {}, locale=be, timezone=Europe/San_Marino
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=104368592,total=517996544
[junit4] 2> NOTE: All tests run in this JVM: [SolrPluginUtilsTest, SolrMetricReporterTest, TestReloadAndDeleteDocs, TestLegacyFieldReuse, OutputWriterTest, SolrIndexConfigTest, TestSchemaVersionResource, SolrCLIZkUtilsTest, DistributedQueryElevationComponentTest, TestNumericTokenStream, TestRandomFaceting, SpellingQueryConverterTest, TestBackupRepositoryFactory, EchoParamsTest, TestJavabinTupleStreamParser, MigrateRouteKeyTest, TestSQLHandler, TestFieldCacheReopen, TestImpersonationWithHadoopAuth, TestDocumentBuilder, BasicDistributedZkTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestXmlQParserPlugin, MBeansHandlerTest, DeleteLastCustomShardedReplicaTest, CurrencyFieldTypeTest, TestExportWriter, TolerantUpdateProcessorTest, CoreMergeIndexesAdminHandlerTest, TestSubQueryTransformer, ZkNodePropsTest, DistribDocExpirationUpdateProcessorTest, BitVectorTest, TestMaxTokenLenTokenizer, TestDocSet, TestRemoteStreaming, DebugComponentTest, PeerSyncTest, TestCollationField, BlobRepositoryCloudTest, SuggesterWFSTTest, BadCopyFieldTest, TestRuleBasedAuthorizationPlugin, TestMergePolicyConfig, HdfsRecoverLeaseTest, TestDistribIDF, SmileWriterTest, OpenExchangeRatesOrgProviderTest, TestCSVResponseWriter, CloudExitableDirectoryReaderTest, TestManagedResource, MetricUtilsTest, ReplicaListTransformerTest, BlockDirectoryTest, TestDownShardTolerantSearch, TestCustomStream, TestNestedDocsSort, SpellCheckCollatorTest, TestPHPSerializedResponseWriter, TestInPlaceUpdatesDistrib, UpdateParamsTest, TestExtendedDismaxParser, SynonymTokenizerTest, PropertiesRequestHandlerTest, MetricsConfigTest, TestSolrCoreSnapshots, TestGraphTermsQParserPlugin, TestFunctionQuery, TestSolrQueryParser, TestLazyCores, CdcrBootstrapTest, WrapperMergePolicyFactoryTest, HLLSerializationTest, TestDynamicFieldCollectionResource, ImplicitSnitchTest, OverseerStatusTest, TestConfigSetProperties, CollectionTooManyReplicasTest, ReplicationFactorTest, TestSolrCLIRunExample, TestStressLucene, TestCustomSort, LeaderElectionIntegrationTest, TestFieldCacheVsDocValues, TestUnifiedSolrHighlighter, BasicZkTest, DocumentAnalysisRequestHandlerTest, TestFieldResource, UUIDFieldTest, FieldAnalysisRequestHandlerTest, SyncSliceTest, CursorPagingTest, HighlighterConfigTest, TestConfigSetsAPI, SolrRequestParserTest, TestSolr4Spatial, TestHighlightDedupGrouping, HdfsUnloadDistributedZkTest, HdfsLockFactoryTest, TestCrossCoreJoin, RAMDirectoryFactoryTest, HdfsChaosMonkeySafeLeaderTest, ResponseHeaderTest, TestOmitPositions, TestLegacyNumericRangeQueryBuilder, TestDocBasedVersionConstraints, ClassificationUpdateProcessorTest, TestManagedStopFilterFactory, TestPerFieldSimilarityWithDefaultOverride, TestCSVLoader, TestTolerantUpdateProcessorRandomCloud, PluginInfoTest, TestLMDirichletSimilarityFactory, TestComponentsName, CursorMarkTest, TestClusterStateMutator, SolrCloudReportersTest, TestSimpleQParserPlugin, TestSolrDeletionPolicy1, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [521/728 (3!)] on J0 in 278.40s, 1 test, 1 failure <<< FAILURES!
[...truncated 647 lines...]
[junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20170720_144626_0505370345072770947752.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid20403.hprof ...
[junit4] Heap dump file created [473267819 bytes in 11.537 secs]
[junit4] <<< JVM J2: EOF ----
[...truncated 7520 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid20403.hprof
Total time: 227 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1357 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1357/
4 tests failed.
FAILED: org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates
Error Message:
Timeout while trying to assert number of documents @ source_collection
Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
at __randomizedtesting.SeedInfo.seed([5BF0D7E98ED145AC:88F987F7CB42D93B]:0)
at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:282)
at org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:233)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: expected:<270> but was:<280>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
... 42 more
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([5BF0D7E98ED145AC:D3A4E833202D2854]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([5BF0D7E98ED145AC:D3A4E833202D2854]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([5BF0D7E98ED145AC:D3A4E833202D2854]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11694 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/init-core-data-001
[junit4] 2> 0 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 411 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.e.j.u.log Logging initialized @9058ms
[junit4] 2> 445 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2007 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /wop/
[junit4] 2> 4473 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 6796 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 8165 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
[junit4] 2> 8310 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 8624 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 8845 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_42444_hdfs____.il297k/webapp
[junit4] 2> 11264 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:42444
[junit4] 2> 14693 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 14763 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 14943 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37802_datanode____.knjnwj/webapp
[junit4] 2> 16693 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37802
[junit4] 2> 19917 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 19919 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 19993 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_45554_datanode____ydaqje/webapp
[junit4] 2> 21885 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[5BF0D7E98ED145AC]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45554
[junit4] 2> 28660 INFO (IPC Server handler 5 on 45037) [ ] BlockStateChange BLOCK* processReport: from storage DS-998161bd-a3ef-4733-89a0-7141260eefe7 node DatanodeRegistration(127.0.0.1:41921, datanodeUuid=527329ec-6199-467e-99ff-edddb015e27a, infoPort=34246, infoSecurePort=0, ipcPort=37474, storageInfo=lv=-56;cid=testClusterID;nsid=101181751;c=0), blocks: 0, hasStaleStorage: true, processing time: 88 msecs
[junit4] 2> 28676 INFO (IPC Server handler 5 on 45037) [ ] BlockStateChange BLOCK* processReport: from storage DS-41feb6d5-0a22-4079-95d8-2559fdef4d1f node DatanodeRegistration(127.0.0.1:41921, datanodeUuid=527329ec-6199-467e-99ff-edddb015e27a, infoPort=34246, infoSecurePort=0, ipcPort=37474, storageInfo=lv=-56;cid=testClusterID;nsid=101181751;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 28660 INFO (IPC Server handler 6 on 45037) [ ] BlockStateChange BLOCK* processReport: from storage DS-5cc70dcd-db3b-4b05-af6e-c1ceac762566 node DatanodeRegistration(127.0.0.1:34554, datanodeUuid=ab5b329c-f7fd-41cb-8043-4ec87b21a7cf, infoPort=44025, infoSecurePort=0, ipcPort=33793, storageInfo=lv=-56;cid=testClusterID;nsid=101181751;c=0), blocks: 0, hasStaleStorage: true, processing time: 13 msecs
[junit4] 2> 28692 INFO (IPC Server handler 6 on 45037) [ ] BlockStateChange BLOCK* processReport: from storage DS-f7a0b1c4-ea11-439a-b178-3afcb82e609e node DatanodeRegistration(127.0.0.1:34554, datanodeUuid=ab5b329c-f7fd-41cb-8043-4ec87b21a7cf, infoPort=44025, infoSecurePort=0, ipcPort=33793, storageInfo=lv=-56;cid=testClusterID;nsid=101181751;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 30586 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 30665 INFO (Thread-112) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 30665 INFO (Thread-112) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 31306 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.ZkTestServer start zk server on port:37746
[junit4] 2> 31939 ERROR (Thread-112) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 32340 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d5afd1b9b0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 32592 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 32657 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 32689 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 32705 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 32719 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 32722 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 32737 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 32739 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 32771 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 32802 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 32833 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 33150 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 34895 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 35220 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1bdb128{/wop,null,AVAILABLE}
[junit4] 2> 35351 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3684686a{SSL,[ssl, http/1.1]}{127.0.0.1:36836}
[junit4] 2> 35351 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.Server Started @44016ms
[junit4] 2> 35351 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://lucene2-us-west.apache.org:45037/hdfs__lucene2-us-west.apache.org_45037__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001_tempDir-002_control_data, replicaType=NRT, hostContext=/wop, hostPort=36836, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/control-001/cores}
[junit4] 2> 35497 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 35498 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 35498 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 35498 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 35566 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-19T13:15:27.559Z
[junit4] 2> 35662 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 35682 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 35684 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/control-001/solr.xml
[junit4] 2> 35998 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 36974 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37746/solr
[junit4] 2> 38875 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 38889 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36836_wop
[junit4] 2> 38891 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.c.Overseer Overseer (id=98334810008387589-127.0.0.1:36836_wop-n_0000000000) starting
[junit4] 2> 39291 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36836_wop
[junit4] 2> 39478 INFO (OverseerStateUpdate-98334810008387589-127.0.0.1:36836_wop-n_0000000000) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 41155 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 41186 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 41187 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 41611 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:36836_wop ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/control-001/cores
[junit4] 2> 42452 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 42547 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37746/solr ready
[junit4] 2> 44749 INFO (qtp38051757-201) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36836_wop&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 44796 INFO (OverseerThreadFactory-6-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 45449 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 45467 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 45849 INFO (zkCallback-9-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 46885 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 47673 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 48257 WARN (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.CurrencyField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 50497 WARN (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.GeoHashField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 50543 WARN (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.LatLonType]. Please consult documentation how to replace it accordingly.
[junit4] 2> 50726 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 52451 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 52516 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 52565 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home
[junit4] 2> 52565 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 52565 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 52596 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 52658 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 52782 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 52782 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 52782 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 55199 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 55265 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 55820 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 55908 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 55909 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 56053 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 56069 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5984755050095497]
[junit4] 2> 57234 INFO (IPC Server handler 7 on 45037) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41921 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5cc70dcd-db3b-4b05-af6e-c1ceac762566:NORMAL:127.0.0.1:34554|RBW], ReplicaUC[[DISK]DS-998161bd-a3ef-4733-89a0-7141260eefe7:NORMAL:127.0.0.1:41921|FINALIZED]]} size 0
[junit4] 2> 57244 INFO (IPC Server handler 0 on 45037) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34554 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5cc70dcd-db3b-4b05-af6e-c1ceac762566:NORMAL:127.0.0.1:34554|RBW], ReplicaUC[[DISK]DS-998161bd-a3ef-4733-89a0-7141260eefe7:NORMAL:127.0.0.1:41921|FINALIZED]]} size 0
[junit4] 2> 58294 WARN (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 59803 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 59803 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 59803 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 59969 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 59969 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 60001 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14701342220532665]
[junit4] 2> 60334 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@b9fa5f1[control_collection_shard1_replica_n1] main]
[junit4] 2> 60386 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 60459 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 60594 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 61238 INFO (searcherExecutor-9-thread-1-processing-n:127.0.0.1:36836_wop x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@b9fa5f1[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 61276 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573356991486623744
[junit4] 2> 61364 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 61364 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 61364 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36836/wop/control_collection_shard1_replica_n1/
[junit4] 2> 61451 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 61451 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:36836/wop/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 61451 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 61506 INFO (zkCallback-9-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 61576 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36836/wop/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 61697 INFO (zkCallback-9-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 61720 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 61775 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=16358
[junit4] 2> 61918 INFO (zkCallback-9-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 62170 INFO (qtp38051757-201) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 62215 INFO (qtp38051757-201) [n:127.0.0.1:36836_wop ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36836_wop&wt=javabin&version=2} status=0 QTime=17607
[junit4] 2> 62463 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 62515 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37746/solr ready
[junit4] 2> 62517 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 62559 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 62695 INFO (OverseerThreadFactory-6-thread-2-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 62708 INFO (OverseerCollectionConfigSetProcessor-98334810008387589-127.0.0.1:36836_wop-n_0000000000) [n:127.0.0.1:36836_wop ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 62708 WARN (OverseerThreadFactory-6-thread-2-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 63205 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 63205 INFO (qtp38051757-199) [n:127.0.0.1:36836_wop ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=646
[junit4] 2> 64069 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-1-001 of type NRT
[junit4] 2> 64091 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 64092 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c4380a3{/wop,null,AVAILABLE}
[junit4] 2> 64092 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@59d56531{SSL,[ssl, http/1.1]}{127.0.0.1:33269}
[junit4] 2> 64093 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.Server Started @72757ms
[junit4] 2> 64093 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://lucene2-us-west.apache.org:45037/hdfs__lucene2-us-west.apache.org_45037__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001_tempDir-002_jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/wop, hostPort=33269, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-1-001/cores}
[junit4] 2> 64171 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 64171 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 64171 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 64171 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 64171 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-19T13:15:56.214Z
[junit4] 2> 64300 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 64300 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-1-001/solr.xml
[junit4] 2> 64460 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 64469 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37746/solr
[junit4] 2> 64618 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 64621 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 64637 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33269_wop
[junit4] 2> 64652 INFO (zkCallback-16-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 64744 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:33269_wop) [n:127.0.0.1:33269_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 64746 INFO (zkCallback-9-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 64747 INFO (OverseerCollectionConfigSetProcessor-98334810008387589-127.0.0.1:36836_wop-n_0000000000) [n:127.0.0.1:36836_wop ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 65570 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 65651 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 65652 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 65654 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33269_wop ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-1-001/cores
[junit4] 2> 66014 INFO (qtp38051757-201) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33269_wop&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 66148 INFO (OverseerThreadFactory-6-thread-3-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33269_wop for creating new replica
[junit4] 2> 66728 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 66839 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 68049 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 68265 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 69240 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 69450 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 69451 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 69452 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home
[junit4] 2> 69452 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 69452 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 69452 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 69463 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 69546 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 69546 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 69689 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 69718 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 69946 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 69969 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 69969 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 70047 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 70048 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5984755050095497]
[junit4] 2> 70499 INFO (IPC Server handler 2 on 45037) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41921 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5cc70dcd-db3b-4b05-af6e-c1ceac762566:NORMAL:127.0.0.1:34554|RBW], ReplicaUC[[DISK]DS-41feb6d5-0a22-4079-95d8-2559fdef4d1f:NORMAL:127.0.0.1:41921|RBW]]} size 0
[junit4] 2> 70516 INFO (IPC Server handler 9 on 45037) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34554 is added to blk_1073741826_1002 size 72
[junit4] 2> 70564 WARN (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 70855 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 70855 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 70855 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 71012 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 71012 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 71016 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14701342220532665]
[junit4] 2> 71320 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@5f7f17cc[collection1_shard2_replica_n0] main]
[junit4] 2> 71328 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 71329 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 71330 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 71338 INFO (searcherExecutor-20-thread-1-processing-n:127.0.0.1:33269_wop x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@5f7f17cc[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 71338 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573357002037395456
[junit4] 2> 71392 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 71392 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 71393 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33269/wop/collection1_shard2_replica_n0/
[junit4] 2> 71393 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 71393 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy https://127.0.0.1:33269/wop/collection1_shard2_replica_n0/ has no replicas
[junit4] 2> 71405 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 71462 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33269/wop/collection1_shard2_replica_n0/ shard2
[junit4] 2> 71625 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 71643 INFO (qtp1588738711-251) [n:127.0.0.1:33269_wop c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4914
[junit4] 2> 71661 INFO (qtp38051757-201) [n:127.0.0.1:36836_wop ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33269_wop&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=5646
[junit4] 2> 72226 INFO (OverseerCollectionConfigSetProcessor-98334810008387589-127.0.0.1:36836_wop-n_0000000000) [n:127.0.0.1:36836_wop ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 72249 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-2-001 of type NRT
[junit4] 2> 72278 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 72280 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20284374{/wop,null,AVAILABLE}
[junit4] 2> 72281 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@69e6d713{SSL,[ssl, http/1.1]}{127.0.0.1:33432}
[junit4] 2> 72281 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.e.j.s.Server Started @80946ms
[junit4] 2> 72281 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://lucene2-us-west.apache.org:45037/hdfs__lucene2-us-west.apache.org_45037__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001_tempDir-002_jetty2, solrconfig=solrconfig.xml, hostContext=/wop, hostPort=33432, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-2-001/cores}
[junit4] 2> 72282 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 72360 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 72360 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 72363 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 72363 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-19T13:16:04.406Z
[junit4] 2> 72434 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 72434 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-2-001/solr.xml
[junit4] 2> 72459 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 72488 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37746/solr
[junit4] 2> 72633 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 72650 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 72665 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33432_wop
[junit4] 2> 72666 INFO (zkCallback-27-thread-1-processing-n:127.0.0.1:33432_wop) [n:127.0.0.1:33432_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 72677 INFO (zkCallback-16-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 72679 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:33269_wop) [n:127.0.0.1:33269_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 72679 INFO (zkCallback-9-thread-1-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 73271 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 73334 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 73335 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 73349 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[5BF0D7E98ED145AC]) [n:127.0.0.1:33432_wop ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-2-001/cores
[junit4] 2> 73660 INFO (qtp38051757-194) [n:127.0.0.1:36836_wop ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33432_wop&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 73743 INFO (OverseerThreadFactory-6-thread-4-processing-n:127.0.0.1:36836_wop) [n:127.0.0.1:36836_wop ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33432_wop for creating new replica
[junit4] 2> 74606 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n0&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 74606 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 75745 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 75869 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.s.IndexSchema [collection1_shard3_replica_n0] Schema name=test
[junit4] 2> 76627 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 77008 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 77021 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n0' (registry 'solr.core.collection1.shard3.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14164317
[junit4] 2> 77021 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home
[junit4] 2> 77021 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 77021 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 77022 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.SolrCore [[collection1_shard3_replica_n0] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_5BF0D7E98ED145AC-001/shard-2-001/cores/collection1_shard3_replica_n0], dataDir=[hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node2/data/]
[junit4] 2> 77023 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
[junit4] 2> 77055 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 77055 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 77208 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 77224 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node2/data
[junit4] 2> 77448 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:45037/solr_hdfs_home/collection1/core_node2/data/index
[junit4] 2> 77455 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 77455 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 77497 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 77501 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5984755050095497]
[junit4] 2> 77941 INFO (IPC Server handler 8 on 45037) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34554 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-41feb6d5-0a22-4079-95d8-2559fdef4d1f:NORMAL:127.0.0.1:41921|RBW], ReplicaUC[[DISK]DS-5cc70dcd-db3b-4b05-af6e-c1ceac762566:NORMAL:127.0.0.1:34554|FINALIZED]]} size 0
[junit4] 2> 77945 INFO (IPC Server handler 0 on 45037) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41921 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5cc70dcd-db3b-4b05-af6e-c1ceac762566:NORMAL:127.0.0.1:34554|FINALIZED], ReplicaUC[[DISK]DS-998161bd-a3ef-4733-89a0-7141260eefe7:NORMAL:127.0.0.1:41921|FINALIZED]]} size 0
[junit4] 2> 78060 WARN (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 78533 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 78595 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 78623 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 78710 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 78725 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 78806 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14701342220532665]
[junit4] 2> 79132 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@68b9d4bc[collection1_shard3_replica_n0] main]
[junit4] 2> 79134 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 79135 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 79164 INFO (qtp631949391-291) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 79198 INFO (searcherExecutor-31-thread-1-processing-n:127.0.0.1:33432_wop x:collection1_shard3_replica_n0 s:shard3 c:collection1) [n:127.0.0.1:33432_wop c:collection1 s:shard3 x:collection1_shard3_replica_n0] o.a.s.c.SolrCore [collection1_shard3_replica_n0] Registered new searcher Searcher@68b9d
[...truncated too long message...]
Interceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_5BF0D7E98ED145AC-001
[junit4] 2> Jul 19, 2017 2:42:48 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=bg-BG, timezone=America/Bahia
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=124922040,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [HdfsChaosMonkeySafeLeaderTest, TestSolrQueryParser, MetricUtilsTest, TestComponentsName, TestLegacyNumericRangeQueryBuilder, TestJsonRequest, TestExpandComponent, TestJavabinTupleStreamParser, TestReplicationHandlerBackup, StatsReloadRaceTest, TestJoin, TestAnalyzeInfixSuggestions, TestRawTransformer, TestRemoteStreaming, UUIDFieldTest, TestInPlaceUpdatesDistrib, DistributedSpellCheckComponentTest, AssignTest, MoveReplicaHDFSUlogDirTest, TestDistributedSearch, RecoveryAfterSoftCommitTest, TestConfigSetsAPI, TestSolr4Spatial, SolrCLIZkUtilsTest, IndexBasedSpellCheckerTest, PKIAuthenticationIntegrationTest, ReplicationFactorTest, MBeansHandlerTest, TestTolerantUpdateProcessorRandomCloud, TestFaceting, TestReloadAndDeleteDocs, LeaderFailureAfterFreshStartTest, SpellCheckCollatorTest, TestTrackingShardHandlerFactory, TestManagedSchemaAPI, DocumentAnalysisRequestHandlerTest, MigrateRouteKeyTest, TestManagedResource, TestMultiWordSynonyms, XmlUpdateRequestHandlerTest, SolrGraphiteReporterTest, BlobRepositoryCloudTest, HdfsLockFactoryTest, TolerantUpdateProcessorTest, BasicZkTest, BadCopyFieldTest, PropertiesRequestHandlerTest, ResponseHeaderTest, ImplicitSnitchTest, TestSolrCoreParser, TestNumericTokenStream, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestDistribIDF, HdfsUnloadDistributedZkTest, TestRandomFaceting, TestExportWriter, ConnectionReuseTest, TestCSVLoader, TestMacroExpander, DebugComponentTest, TestSchemaManager, DeleteShardTest, TestConfigSetProperties, TestLazyCores, TestNumericTerms32, TestSubQueryTransformer, TestPayloadCheckQParserPlugin, TestXmlQParserPlugin, RankQueryTest, BlockDirectoryTest, DistributedQueryElevationComponentTest, TestLMDirichletSimilarityFactory, ExitableDirectoryReaderTest, TestCloudJSONFacetJoinDomain, TestStressLucene, SolrCoreTest, ExternalFileFieldSortTest, RAMDirectoryFactoryTest, TestSchemaNameResource, TestSolrCloudWithSecureImpersonation, FieldAnalysisRequestHandlerTest, AtomicUpdatesTest, DistribDocExpirationUpdateProcessorTest, HighlighterConfigTest, DistributedVersionInfoTest, OverseerCollectionConfigSetProcessorTest, TestStressUserVersions, TestFastOutputStream, SyncSliceTest, TestConfigsApi, TestNestedDocsSort, TestSolrDeletionPolicy1, TestDocBasedVersionConstraints, TestSizeLimitedDistributedMap, TestCustomSort, TestDocSet, V2StandaloneTest, TestEmbeddedSolrServerConstructors, TestMinMaxOnMultiValuedField, TestPhraseSuggestions, ZkNodePropsTest, TestLuceneMatchVersion, TestFunctionQuery, TestIBSimilarityFactory, TestMultiValuedNumericRangeQuery, ChaosMonkeySafeLeaderWithPullReplicasTest, TestMaxTokenLenTokenizer, TestDocumentBuilder, TestUseDocValuesAsStored2, TestRandomCollapseQParserPlugin, HdfsBasicDistributedZk2Test, TestFieldCacheReopen, CursorMarkTest, DistributedExpandComponentTest, TestGraphTermsQParserPlugin, EnumFieldTest, CreateCollectionCleanupTest, AddBlockUpdateTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, TestReplicationHandler, QueryElevationComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, BasicFunctionalityTest, TestBadConfig, ShowFileRequestHandlerTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterTest, SuggesterWFSTTest, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, CacheHeaderTest, LukeRequestHandlerTest, TestQueryTypes, PrimitiveFieldTypeTest, TestOmitPositions, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestCollationField, ReturnFieldsTest, QueryParsingTest, JsonLoaderTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, TestBinaryField, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, SpellPossibilityIteratorTest, TestCodecSupport, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, CdcrRequestHandlerTest, CollectionTooManyReplicasTest, ConcurrentDeleteAndCreateCollectionTest, DeleteNodeTest, DistribJoinFromCollectionTest, ForceLeaderTest, MultiThreadedOCPTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [549/728 (4!)] on J0 in 236.10s, 1 test, 1 failure <<< FAILURES!
[...truncated 43855 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1356 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1356/
6 tests failed.
FAILED: org.apache.lucene.index.TestIndexSorting.testRandom3
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([23B684C04AACCABB]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.index.TestIndexSorting
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([23B684C04AACCABB]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([3F511470C5415E2:8BA12E9DA2A8781A]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([3F511470C5415E2:8BA12E9DA2A8781A]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
6 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=121704, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=121703, name=StoppableCommitThread-SendThread(127.0.0.1:39461), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 3) Thread[id=121723, name=zkCallback-31916-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=121705, name=zkCallback-31916-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=121662, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55) 6) Thread[id=121724, name=zkCallback-31916-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=121704, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
2) Thread[id=121703, name=StoppableCommitThread-SendThread(127.0.0.1:39461), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
3) Thread[id=121723, name=zkCallback-31916-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=121705, name=zkCallback-31916-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=121662, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
6) Thread[id=121724, name=zkCallback-31916-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3F511470C5415E2]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=121703, name=StoppableCommitThread-SendThread(127.0.0.1:39461), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 2) Thread[id=121723, name=zkCallback-31916-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=121705, name=zkCallback-31916-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=121724, name=zkCallback-31916-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=121703, name=StoppableCommitThread-SendThread(127.0.0.1:39461), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
2) Thread[id=121723, name=zkCallback-31916-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=121705, name=zkCallback-31916-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=121724, name=zkCallback-31916-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3F511470C5415E2]:0)
Build Log:
[...truncated 1723 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexSorting
[junit4] 2> Jul 16, 2017 6:31:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexSorting
[junit4] 2> 1) Thread[id=856, name=SUITE-TestIndexSorting-seed#[23B684C04AACCABB], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4] 2> 2) Thread[id=857, name=TEST-TestIndexSorting.testRandom3-seed#[23B684C04AACCABB], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at sun.nio.ch.NativeThread.current(Native Method)
[junit4] 2> at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
[junit4] 2> at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:284)
[junit4] 2> at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:50)
[junit4] 2> at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
[junit4] 2> at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
[junit4] 2> at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.readInternal(SimpleFSDirectory.java:171)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:298)
[junit4] 2> at org.apache.lucene.util.packed.DirectReader$DirectPackedReader32.get(DirectReader.java:267)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$13.get(Lucene70DocValuesProducer.java:643)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$22.nextValue(Lucene70DocValuesProducer.java:1271)
[junit4] 2> at org.apache.lucene.index.AssertingLeafReader$AssertingSortedNumericDocValues.nextValue(AssertingLeafReader.java:752)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextValue(DocValuesConsumer.java:446)
[junit4] 2> at org.apache.lucene.codecs.asserting.AssertingDocValuesFormat$AssertingDocValuesConsumer.addSortedNumericField(AssertingDocValuesFormat.java:162)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
[junit4] 2> at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] 2> at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
[junit4] 2> at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
[junit4] 2> at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2300)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=23B684C04AACCABB -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=de-DE -Dtests.timezone=AGT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 7200s J1 | TestIndexSorting.testRandom3 <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([23B684C04AACCABB]:0)
[junit4] 2> Jul 16, 2017 6:31:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> Jul 16, 2017 6:32:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestIndexSorting:
[junit4] 2> 1) Thread[id=857, name=TEST-TestIndexSorting.testRandom3-seed#[23B684C04AACCABB], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at sun.nio.ch.NativeThread.current(Native Method)
[junit4] 2> at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
[junit4] 2> at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:284)
[junit4] 2> at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:50)
[junit4] 2> at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
[junit4] 2> at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
[junit4] 2> at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.readInternal(SimpleFSDirectory.java:171)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:298)
[junit4] 2> at org.apache.lucene.util.packed.DirectReader$DirectPackedReader28.get(DirectReader.java:248)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$13.get(Lucene70DocValuesProducer.java:643)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$22.nextValue(Lucene70DocValuesProducer.java:1271)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextValue(DocValuesConsumer.java:446)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeValues(Lucene70DocValuesConsumer.java:162)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedNumericField(Lucene70DocValuesConsumer.java:530)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
[junit4] 2> at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] 2> at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
[junit4] 2> at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
[junit4] 2> at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2300)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Jul 16, 2017 6:32:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=857, name=TEST-TestIndexSorting.testRandom3-seed#[23B684C04AACCABB], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1/temp/lucene.index.TestIndexSorting_23B684C04AACCABB-001
[junit4] 2> Jul 16, 2017 9:32:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Memory)}, docValues:{multi_valued_double=DocValuesFormat(name=Direct), multi_valued_long=DocValuesFormat(name=Lucene70), bytes=DocValuesFormat(name=Direct), double=DocValuesFormat(name=Direct), foo=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Direct), multi_valued_int=DocValuesFormat(name=Asserting), float=DocValuesFormat(name=Memory), multi_valued_bytes=DocValuesFormat(name=Direct), multi_valued_float=DocValuesFormat(name=Lucene70), int=DocValuesFormat(name=Direct), long=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=666, maxMBSortInHeap=6.671493144041433, sim=RandomSimilarity(queryNorm=true): {}, locale=de-DE, timezone=AGT
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=85344528,total=524812288
[junit4] 2> NOTE: All tests run in this JVM: [TestAllFilesDetectTruncation, TestAddIndexes, TestElevationComparator, TestSparseFixedBitSet, TestIndexWriterOnJRECrash, LimitedFiniteStringsIteratorTest, TestSegmentInfos, TestFixedBitDocIdSet, TestParallelReaderEmptyIndex, TestFileSwitchDirectory, TestOmitTf, TestFastDecompressionMode, TestByteSlices, TestSpanContainQuery, TestCrash, TestToken, TestPrefixCodedTerms, TestRecyclingByteBlockAllocator, TestMultiTermConstantScore, TestRateLimiter, TestInfoStream, TestFilterDirectoryReader, TestTermVectorsReader, TestSearcherManager, TestCloseableThreadLocal, TestGraphTokenizers, TestMixedDocValuesUpdates, TestSameScoresWithThreads, TestIndexSorting]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.seed=23B684C04AACCABB -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=de-DE -Dtests.timezone=AGT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | TestIndexSorting (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([23B684C04AACCABB]:0)
[junit4] Completed [408/454 (1!)] on J1 in 7220.15s, 2 tests, 2 errors <<< FAILURES!
[...truncated 9870 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/init-core-data-001
[junit4] 2> 336906 WARN (SUITE-ChaosMonkeySafeLeaderTest-seed#[3F511470C5415E2]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 336906 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[3F511470C5415E2]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 336907 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[3F511470C5415E2]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 336907 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[3F511470C5415E2]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /ey/ir
[junit4] 2> 336913 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 336914 INFO (Thread-627) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 336914 INFO (Thread-627) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 336915 ERROR (Thread-627) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 337014 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.ZkTestServer start zk server on port:36542
[junit4] 2> 337026 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 337027 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 337028 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 337029 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 337030 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 337030 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 337031 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 337032 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 337033 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 337034 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 337034 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 337038 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 337131 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 337132 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2bf7ae86{/ey/ir,null,AVAILABLE}
[junit4] 2> 337134 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2c151023{SSL,[ssl, http/1.1]}{127.0.0.1:60865}
[junit4] 2> 337135 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.Server Started @339656ms
[junit4] 2> 337135 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/tempDir-001/control/data, replicaType=NRT, hostContext=/ey/ir, hostPort=60865, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/control-001/cores}
[junit4] 2> 337136 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 337136 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 337136 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 337136 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 337136 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-16T11:15:55.780Z
[junit4] 2> 337142 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 337142 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/control-001/solr.xml
[junit4] 2> 337147 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 337150 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36542/solr
[junit4] 2> 337153 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d4b1ca8260004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 337226 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 337227 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60865_ey%2Fir
[junit4] 2> 337228 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.Overseer Overseer (id=98317353324249093-127.0.0.1:60865_ey%2Fir-n_0000000000) starting
[junit4] 2> 337234 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60865_ey%2Fir
[junit4] 2> 337241 INFO (zkCallback-489-thread-1-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 337341 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 337345 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 337345 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 337346 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/control-001/cores
[junit4] 2> 337360 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 337361 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36542/solr ready
[junit4] 2> 337386 INFO (qtp824163026-3241) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:60865_ey%252Fir&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 337388 INFO (OverseerThreadFactory-1686-thread-1-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 337495 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 337496 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 337599 INFO (zkCallback-489-thread-1-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 338513 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 338537 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 338616 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 338649 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 338650 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 338680 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 338680 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 338693 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=14.9130859375, floorSegmentMB=0.41015625, forceMergeDeletesPctAllowed=4.865892567341125, segmentsPerTier=6.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 338704 WARN (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 338742 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 338743 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 338743 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 338744 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 338745 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1218563949, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 338746 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@15dc5bc7[control_collection_shard1_replica_n1] main]
[junit4] 2> 338747 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 338747 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 338748 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 338749 INFO (searcherExecutor-1689-thread-1-processing-n:127.0.0.1:60865_ey%2Fir x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@15dc5bc7[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 338749 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573077655112122368
[junit4] 2> 338755 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 338755 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 338755 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:60865/ey/ir/control_collection_shard1_replica_n1/
[junit4] 2> 338755 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 338755 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:60865/ey/ir/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 338755 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 338758 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:60865/ey/ir/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 338859 INFO (zkCallback-489-thread-1-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 338909 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 338912 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1416
[junit4] 2> 338916 INFO (qtp824163026-3241) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 339016 INFO (zkCallback-489-thread-2-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 339389 INFO (OverseerCollectionConfigSetProcessor-98317353324249093-127.0.0.1:60865_ey%2Fir-n_0000000000) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 339916 INFO (qtp824163026-3241) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:60865_ey%252Fir&wt=javabin&version=2} status=0 QTime=2530
[junit4] 2> 339921 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 339922 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36542/solr ready
[junit4] 2> 339922 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 339927 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=4&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 339928 INFO (OverseerThreadFactory-1686-thread-2-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 339928 WARN (OverseerThreadFactory-1686-thread-2-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 340132 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 340132 INFO (qtp824163026-3239) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=4&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=205
[junit4] 2> 340219 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-1-001 of type NRT
[junit4] 2> 340221 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 340222 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@42d84362{/ey/ir,null,AVAILABLE}
[junit4] 2> 340222 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@8094097{SSL,[ssl, http/1.1]}{127.0.0.1:44865}
[junit4] 2> 340223 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.Server Started @342744ms
[junit4] 2> 340223 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/ey/ir, hostPort=44865, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-1-001/cores}
[junit4] 2> 340223 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 340223 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 340223 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 340223 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 340223 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-16T11:15:58.867Z
[junit4] 2> 340226 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 340226 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-1-001/solr.xml
[junit4] 2> 340233 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 340236 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36542/solr
[junit4] 2> 340246 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 340247 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 340249 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44865_ey%2Fir
[junit4] 2> 340250 INFO (zkCallback-489-thread-2-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 340250 INFO (zkCallback-496-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 340251 INFO (zkCallback-501-thread-1-processing-n:127.0.0.1:44865_ey%2Fir) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 340350 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 340354 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 340354 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 340355 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-1-001/cores
[junit4] 2> 340400 INFO (qtp761555604-3283) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:44865_ey%252Fir&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 340402 INFO (OverseerThreadFactory-1686-thread-3-processing-n:127.0.0.1:60865_ey%2Fir) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:44865_ey%2Fir for creating new replica
[junit4] 2> 340402 INFO (OverseerCollectionConfigSetProcessor-98317353324249093-127.0.0.1:60865_ey%2Fir-n_0000000000) [n:127.0.0.1:60865_ey%2Fir ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 340405 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 340405 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 341419 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 341430 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 341486 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 341504 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 341504 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@487db3e4
[junit4] 2> 341504 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 341504 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-1-001/cores/collection1_shard2_replica_n0/data/]
[junit4] 2> 341507 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=14.9130859375, floorSegmentMB=0.41015625, forceMergeDeletesPctAllowed=4.865892567341125, segmentsPerTier=6.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 341591 WARN (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 341627 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 341628 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 341628 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 341628 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 341630 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1218563949, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 341631 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@50b61261[collection1_shard2_replica_n0] main]
[junit4] 2> 341632 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 341632 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 341632 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 341633 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1573077658136215552
[junit4] 2> 341634 INFO (searcherExecutor-1700-thread-1-processing-n:127.0.0.1:44865_ey%2Fir x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@50b61261[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 341637 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 341637 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 341637 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:44865/ey/ir/collection1_shard2_replica_n0/
[junit4] 2> 341637 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 341637 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy https://127.0.0.1:44865/ey/ir/collection1_shard2_replica_n0/ has no replicas
[junit4] 2> 341637 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 341640 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:44865/ey/ir/collection1_shard2_replica_n0/ shard2
[junit4] 2> 341791 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 341793 INFO (qtp761555604-3281) [n:127.0.0.1:44865_ey%2Fir c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1388
[junit4] 2> 341796 INFO (qtp761555604-3283) [n:127.0.0.1:44865_ey%2Fir ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:44865_ey%252Fir&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1395
[junit4] 2> 341889 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-2-001 of type NRT
[junit4] 2> 341891 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 341892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@384ca277{/ey/ir,null,AVAILABLE}
[junit4] 2> 341892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2c5a2b8f{SSL,[ssl, http/1.1]}{127.0.0.1:53348}
[junit4] 2> 341892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.e.j.s.Server Started @344413ms
[junit4] 2> 341892 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/ey/ir, hostPort=53348, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_3F511470C5415E2-001/shard-2-001/cores}
[junit4] 2> 341893 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 341893 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 341893 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 341893 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[3F511470C5415E2]) [ ] o.a.s.s.SolrDispat
[...truncated too long message...]
tedDistributedMap, TestFunctionQuery, TestConfigSetProperties, TestBadConfig, EnumFieldTest, TestRawTransformer, CdcrBootstrapTest, DebugComponentTest, TestFaceting, DistribDocExpirationUpdateProcessorTest, TestInfoStreamLogging, CdcrRequestHandlerTest, V2StandaloneTest, TestFieldCacheReopen, HdfsUnloadDistributedZkTest, TestCrossCoreJoin, TestNumericTokenStream, TestExportWriter, TestCollationField, TestCollapseQParserPlugin, BadCopyFieldTest, TestReloadAndDeleteDocs, TestConfig, ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderWithPullReplicasTest -Dtests.seed=3F511470C5415E2 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=he -Dtests.timezone=CET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | ChaosMonkeySafeLeaderWithPullReplicasTest (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
[junit4] > 1) Thread[id=121704, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > 2) Thread[id=121703, name=StoppableCommitThread-SendThread(127.0.0.1:39461), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 3) Thread[id=121723, name=zkCallback-31916-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=121705, name=zkCallback-31916-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=121662, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
[junit4] > 6) Thread[id=121724, name=zkCallback-31916-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([3F511470C5415E2]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=121703, name=StoppableCommitThread-SendThread(127.0.0.1:39461), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 2) Thread[id=121723, name=zkCallback-31916-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=121705, name=zkCallback-31916-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=121724, name=zkCallback-31916-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([3F511470C5415E2]:0)
[junit4] Completed [215/728 (2!)] on J0 in 87.11s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 44820 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1355 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1355/
2 tests failed.
FAILED: org.apache.solr.core.TestJmxIntegration.testJmxRegistration
Error Message:
org.apache.lucene.store.AlreadyClosedException: this Directory is closed
Stack Trace:
javax.management.RuntimeMBeanException: org.apache.lucene.store.AlreadyClosedException: this Directory is closed
at __randomizedtesting.SeedInfo.seed([9C9B47172BCB713:8718D04B1FFDEF76]:0)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.rethrow(DefaultMBeanServerInterceptor.java:839)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.rethrowMaybeMBeanException(DefaultMBeanServerInterceptor.java:852)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:651)
at com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
at org.apache.solr.core.TestJmxIntegration.testJmxRegistration(TestJmxIntegration.java:121)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.lucene.store.AlreadyClosedException: this Directory is closed
at org.apache.lucene.store.BaseDirectory.ensureOpen(BaseDirectory.java:51)
at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:233)
at org.apache.solr.core.DirectoryFactory.sizeOfDirectory(DirectoryFactory.java:271)
at org.apache.solr.core.SolrCore.getIndexSize(SolrCore.java:407)
at org.apache.solr.core.SolrCore.lambda$initializeMetrics$7(SolrCore.java:1137)
at com.codahale.metrics.JmxReporter$JmxGauge.getValue(JmxReporter.java:188)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:71)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:275)
at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
at com.sun.jmx.mbeanserver.PerInterface.getAttribute(PerInterface.java:83)
at com.sun.jmx.mbeanserver.MBeanSupport.getAttribute(MBeanSupport.java:206)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:647)
... 41 more
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([9C9B47172BCB713:819D8BABDC40DAEB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11840 lines...]
[junit4] Suite: org.apache.solr.core.TestJmxIntegration
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.core.TestJmxIntegration_9C9B47172BCB713-001/init-core-data-001
[junit4] 2> 1300418 WARN (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=32 numCloses=32
[junit4] 2> 1300418 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1300419 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1300420 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 1300420 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1300440 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1300456 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 1300582 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1300608 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@8102aea
[junit4] 2> 1300611 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@8102aea
[junit4] 2> 1300611 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@8102aea
[junit4] 2> 1300612 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1300646 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1300662 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1300926 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1300956 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 1300959 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@8102aea
[junit4] 2> 1300959 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1300959 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.core.TestJmxIntegration_9C9B47172BCB713-001/init-core-data-001/]
[junit4] 2> 1300969 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=55.9208984375, floorSegmentMB=2.0625, forceMergeDeletesPctAllowed=13.883160332096928, segmentsPerTier=33.0, maxCFSSegmentSizeMB=0.9609375, noCFSRatio=0.5087435382011617
[junit4] 2> 1301135 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1301135 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1301136 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1301136 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1301137 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.9794921875, noCFSRatio=0.692434609545824]
[junit4] 2> 1301137 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@68579f5d[collection1] main]
[junit4] 2> 1301152 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 1301163 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 1301370 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1301370 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 1301370 INFO (coreLoadExecutor-2990-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572999428115005440
[junit4] 2> 1301370 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 1301370 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 1301399 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 1301412 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 1301414 INFO (SUITE-TestJmxIntegration-seed#[9C9B47172BCB713]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 1301430 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 1301430 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 1301430 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 1301430 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 1301444 INFO (searcherExecutor-2991-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@68579f5d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1301446 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[9C9B47172BCB713]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testJmxUpdate
[junit4] 2> 1301564 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[9C9B47172BCB713]) [ ] o.a.s.c.TestJmxIntegration Mbeans in server: [solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=handlerStart, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=max, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=requests, java.lang:type=Threading, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=errors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=requestTimes, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=handlerStart, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=threadPool.httpShardExecutor.duration, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=handlerStart, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=handlerStart, solr:dom1=jvm,name0=buffers,name1=mapped,name=TotalCapacity, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=rollbacks, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=requestTimes, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=clientErrors, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=clientErrors, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmenter,name=gap.requests, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=HIGHLIGHTER,scope=SolrBoundaryScanner,name=default.requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=errors, solr:dom1=jvm,name0=threads,name1=waiting,name=count, solr:dom1=jvm,name0=classes,name=unloaded, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=timeouts, solr:dom1=node,category=CONTAINER,scope=cores,name=loaded, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=instanceDir, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=totalTime, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=new,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=clientErrors, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=threadPool.httpShardExecutor.completed, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=handlerStart, java.lang:type=Runtime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=errors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.duration, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=timeouts, solr:dom1=core,dom2=collection1,category=CORE,scope=fs,name=totalSpace, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=TLOG,name=state, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=handlerStart, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.duration, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/config,name=clientErrors, solr:dom1=node,category=CONTAINER,scope=cores,name=unloaded, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=serverErrors, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/graph,name=clientErrors, solr:dom1=jvm,name0=os,name=totalSwapSpaceSize, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeErrors, java.lang:type=MemoryPool,name=PS Survivor Space, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=errors, solr:dom1=jvm,name0=os,name=availableProcessors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,scope=fs,name=totalSpace, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=errors, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=used, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=SEARCHER,scope=new,name=warmup, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=requests, solr:dom1=jvm,name0=threads,name=count, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=caching, solr:dom1=jvm,name0=memory,name1=total,name=used, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFormatter,name=html.requests, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=handlerStart, solr:dom1=core,dom2=collection1,category=TLOG,scope=replay,name=ops, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=initCount, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=serverErrors, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=handlerStart, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=HIGHLIGHTER,scope=SolrFragmenter,name=default.requests, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreLoadExecutor.duration, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=requestTimes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=handlerStart, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=adds, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=errors, solr:dom1=jvm,name0=os,name=version, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=serverErrors, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=usage, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=isSlave, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json/docs,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=INDEX,name=size, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=deletesByQuery, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=SEARCHER,scope=new,name=time, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=requests, solr:dom1=jvm,name0=os,name=maxFileDescriptorCount, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmentsBuilder,name=simple.requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=confFilesToReplicate, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=cumulativeErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=requestTimes, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=confFilesToReplicate, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=handlerStart, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragListBuilder,name=default.requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=splits, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=clientErrors, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=fetcher, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=timeouts, solr:dom1=jvm,name0=threads,name1=blocked,name=count, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=autoCommits, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=totalTime, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=totalTime, solr:dom1=jvm,name0=gc,name1=PS-MarkSweep,name=time, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=re
[junit4] 2> quests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=SEARCHER,name=new, solr:dom1=node,category=ADMIN,scope=/admin/info,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=handlerStart, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=clientErrors, solr:dom1=jvm,name0=threads,name1=terminated,name=count, java.lang:type=MemoryManager,name=Metaspace Manager, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=errors, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.completed, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=timeouts, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=refCount, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=aliases, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=max, solr:dom1=node,category=ADMIN,scope=/admin/info,name=requestTimes, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=isSlave, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=SEARCHER,scope=new,name=maxReached, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=timeouts, solr:dom1=jvm,name0=os,name=freeSwapSpaceSize, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=requestTimes, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=availableConnections, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=indexPath, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=errors, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=committed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=handlerStart, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=requestTimes, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=used-after-gc, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=timeouts, java.lang:type=GarbageCollector,name=PS Scavenge, solr:dom1=node,category=ADMIN,scope=/admin/info,name=clientErrors, solr:dom1=jvm,name0=memory,name1=heap,name=usage, java.nio:type=BufferPool,name=mapped, com.sun.management:type=DiagnosticCommand, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=used-after-gc, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=replicationEnabled, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=new,name=maxReached, solr:dom1=node,category=ADMIN,scope=/admin/info,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.running, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=merges, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateExecutor.duration, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=requests, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=serverErrors, solr:dom1=node,category=CONTAINER,scope=fs,name=usableSpace, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=serverErrors, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=errors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=merges, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=replicateAfter, solr:dom1=node,category=CONTAINER,scope=version,name=specification, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=errors, solr:dom1=jvm,name0=threads,name1=timed_waiting,name=count, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=errors, solr:dom1=core,dom2=collection1,category=CORE,name=instanceDir, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=clientErrors, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=used, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=serverErrors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=readerDir, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=totalTime, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.submitted, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=errors, solr:dom1=core,dom2=collection1,category=CACHE,scope=core,name=fieldCache, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=SEARCHER,scope=new,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=handlerStart, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=max, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=softAutoCommits, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=totalTime, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=committed, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=handlerStart, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmenter,name=default.requests, solr:dom1=node,category=ADMIN,scope=/admin/info,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=serverErrors, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmentsBuilder,name=default.requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=docsPending, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=adds, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=used-after-gc, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=TLOG,scope=applyingBuffered,name=ops, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=requestTimes, solr:dom1=core,dom2=collection1,category=CORE,name=aliases, solr:dom1=jvm,name0=memory,name1=total,name=max, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=errors, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=totalTime, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=registeredAt, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=usage, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=handlerStart, solr:dom1=node,category=ADMIN,scope=/admin/info,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=handlerStart, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=used, solr:dom1=core,dom2=collection1,category=CORE,name=startTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/graph,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=requestTimes, java.lang:type=MemoryPool,name=PS Old Gen, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=init, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=requestTimes, com.sun.management:type=HotSpotDiagnostic, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=requests, solr:dom1=jvm,name0=memory,name1=non-heap,name=committed, solr:dom1=jvm,name0=buffers,name1=mapped,name=MemoryUsed, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json,name=requests, solr:dom1=jvm,name0=buffers,name1=direct,name=Count, solr:dom1=jvm,name0=os,name=processCpuTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=timeouts, solr:dom1=jvm,name0=memory,name1=heap,name=committed, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=requestTimes, solr:dom1=jvm,name0=threads,name1=deadlock,name=count, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=errors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=warmupTime, solr:dom1=jvm,name0=threads,name1=runnable,name=count, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,scope=fs,name=usableSpace, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=errors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=indexVersion, solr:dom1=jvm,name0=memory,name1=heap,name=init, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=numDocs, solr:dom1=core,dom2=collection1,category=QUERY,
[junit4] 2> scope=/spellCheckWithWordbreak,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/graph,name=timeouts, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFormatter,name=default.requests, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmenter,name=regex.requests, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json/docs,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=commits, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=softAutoCommitMaxTime, java.lang:type=GarbageCollector,name=PS MarkSweep, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/config,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=cumulativeDeletesByQuery, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=requests, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/graph,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=handlerStart, solr:dom1=node,category=CONTAINER,scope=cores,name=lazy, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=optimizes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=totalTime, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=init, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=splits, solr:dom1=jvm,name0=os,name=openFileDescriptorCount, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=pendingConnections, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=committed, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=errors, solr:dom1=node,category=CACHE,name=fieldCache, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=rollbacks, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=expungeDeletes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=collection, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=cumulativeAdds, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=totalTime, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=max, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=totalTime, solr:dom1=jvm,name0=gc,name1=PS-Scavenge,name=count, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=autoCommitMaxTime, solr:dom1=node,category=ADMIN,scope=/admin/info,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/config,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/dump,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json/docs,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=indexDir, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=requests, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=handlerStart, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=requests, solr:dom1=jvm,name0=os,name=systemLoadAverage, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=requests, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.submitted, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=serverErrors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=reader, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=requestTimes, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrBoundaryScanner,name=breakIterator.requests, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=handlerStart, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=usage, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=max, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=init, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json/docs,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/ping,name=timeouts, solr:dom1=core,dom2=collection1,category=TLOG,name=state, solr:dom1=jvm,name0=buffers,name1=direct,name=MemoryUsed, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.submitted, solr:dom1=core,dom2=collection1,category=TLOG,scope=replay,name=remaining.bytes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=handlerStart, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeDeletesById, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeDeletesByQuery, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=requestTimes, java.lang:type=MemoryPool,name=PS Eden Space, solr:dom1=core,dom2=collection1,category=CORE,scope=fs,name=usableSpace, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=clientErrors, solr:dom1=jvm,name0=os,name=totalPhysicalMemorySize, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=init, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=HIGHLIGHTER,scope=SolrFormatter,name=default.requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=errors, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/config,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=serverErrors, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=usage, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=deletesByQuery, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=requestTimes, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=timeouts, solr:dom1=jvm,name0=threads,name1=new,name=count, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=TLOG,scope=replay,name=remaining.bytes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=serverErrors, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=committed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=clientErrors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=openedAt, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=requests, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=leasedConnections, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=softAutoCommits, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/schema,name=totalTime, solr:dom1=jvm,name0=memory,name1=non-heap,name=used, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=generation, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=coreName, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=clientErrors, java.lang:type=MemoryPool,name=Code Cache, solr:dom1=core,dom2=collection1,category=CORE,name=refCount, java.nio:type=BufferPool,name=direct, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=errors, solr:dom1=jvm,name0=threads,name1=daemon,name=count, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=totalTime, solr:dom1=jvm,name0=memory,name1=non-heap,name=init, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=requestTimes, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=queryResultCache, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=totalTime, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=searcherName, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=requests, solr:dom1=jvm,name0=memory,name1=heap,name=max, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/config,name=errors, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=errors, solr:dom1=node,categor
[junit4] 2> y=ADMIN,scope=/admin/cores,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=deletesById, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=indexSize, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=totalTime, solr:dom1=jvm,name0=buffers,name1=mapped,name=Count, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=clientErrors, java.lang:type=Compilation, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=indexSize, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=requests, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateExecutor.submitted, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=committed, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=serverErrors, solr:dom1=core,dom2=collection1,category=TLOG,scope=buffered,name=ops, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=requestTimes, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=pendingConnections, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=documentCache, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=handlerStart, java.lang:type=MemoryPool,name=Compressed Class Space, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=requests, solr:dom1=jvm,name0=os,name=committedVirtualMemorySize, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=errors, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=usage, solr:dom1=jvm,name0=memory,name1=non-heap,name=max, solr:dom1=core,dom2=collection1,category=CORE,name=coreName, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreLoadExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=handlerStart, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=perSegFilter, solr:dom1=jvm,name0=memory,name1=heap,name=used, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=totalTime, solr:dom1=jvm,name0=os,name=systemCpuLoad, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=clientErrors, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=requestTimes, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrBoundaryScanner,name=default.requests, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=totalTime, solr:dom1=core,dom2=collection1,category=TLOG,scope=applyingBuffered,name=ops, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=docsPending, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=totalTime, solr:dom1=jvm,name0=buffers,name1=direct,name=TotalCapacity, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=requestTimes, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=generation, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=errors, java.lang:type=OperatingSystem, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=handlerStart, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=handlerStart, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=requestTimes, solr:dom1=core,dom2=collection1,category=TLOG,scope=replay,name=remaining.logs, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=requestTimes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=requestTimes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.duration, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=totalTime, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=max, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=used, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=serverErrors, solr:dom1=core,dom2=collection1,category=INDEX,name=sizeInBytes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/get,name=errors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=commits, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeAdds, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=isMaster, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=cumulativeDeletesById, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=totalTime, java.lang:type=MemoryManager,name=CodeCacheManager, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/file,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=timeouts, solr:dom1=node,category=QUERY,scope=/admin/autoscaling,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=totalTime, solr:dom1=core,dom2=collection1,category=INDEX,name=size, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/select,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/csv,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/system,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/config,name=timeouts, solr:dom1=jvm,name0=os,name=arch, solr:dom1=jvm,name0=os,name=name, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=init, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/export,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=serverErrors, solr:dom1=jvm,name0=os,name=processCpuLoad, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=maxDoc, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=committed, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=INDEX,name=sizeInBytes, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmentsBuilder,name=scoreOrder.requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=errors, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=replicationEnabled, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=errors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/sql,name=requests, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=deletedDocs, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=TLOG,scope=buffered,name=ops, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/info,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/segments,name=totalTime, JMImplementation:type=MBeanServerDelegate, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/stream,name=errors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.completed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=handlerStart, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=deletesById, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=new,name=time, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=errors, java.lang:type=ClassLoading, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=requestTimes, java.lang:type=MemoryPool,name=Metaspace, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=expungeDeletes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=updateHandler,name=optimizes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json/docs,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=requestTimes, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=fieldValueCache, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=REPLICATION,scope=/replication,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/debug/dump,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=autoCommits, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=requests, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=HIGHLIGHTER,scope=SolrEncoder,name=default.requests, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=handlerStart, solr:dom1=jvm,name0=gc,name1=PS-MarkSweep,name=count, solr:dom1=node,category=CONTAINER,scope=fs,name=totalSpace, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=errors, solr:dom1=jvm,name0=os,name=freePhysicalMemorySize, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=update,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.completed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=clientErrors, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/graph,name=requests, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=new,name=warmup, solr:dom1=core,dom2=c
[junit4] 2> ollection1,category=QUERY,scope=/mlt,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=UPDATE,scope=/update/json/docs,name=requests, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=maxConnections, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/mbeans,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=CORE,name=shard, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=totalTime, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/plugins,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=totalTime, solr:dom1=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=isMaster, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/logging,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=requests, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrEncoder,name=default.requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=requestTimes, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/terms,name=requestTimes, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=used, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=timeouts, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=replicateAfter, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=filterCache, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=QUERY,scope=/graph,name=errors, solr:dom1=jvm,name0=gc,name1=PS-Scavenge,name=time, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreLoadExecutor.completed, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=requestTimes, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrBoundaryScanner,name=simple.requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=TLOG,scope=replay,name=remaining.logs, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/threads,name=serverErrors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=maxConnections, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/properties,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=clientErrors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=availableConnections, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n0,category=ADMIN,scope=/admin/luke,name=handlerStart, solr:dom1=
[...truncated too long message...]
apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_9C9B47172BCB713-001
[junit4] 2> Jul 15, 2017 3:44:07 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=26399, maxDocsPerChunk=9, blockSize=8), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=26399, blockSize=8)), sim=RandomSimilarity(queryNorm=true): {}, locale=id, timezone=EET
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=2,free=97336720,total=517996544
[junit4] 2> NOTE: All tests run in this JVM: [TestManagedSchemaAPI, TestLocalFSCloudBackupRestore, ExternalFileFieldSortTest, TestLeaderElectionWithEmptyReplica, DistributedVersionInfoTest, DeleteLastCustomShardedReplicaTest, TestManagedResource, CdcrRequestHandlerTest, TestSolrCloudWithSecureImpersonation, HdfsSyncSliceTest, TestClusterStateMutator, ChaosMonkeyNothingIsSafeTest, MBeansHandlerTest, TestSimpleQParserPlugin, TestNestedDocsSort, TestAnalyzeInfixSuggestions, TestMultiWordSynonyms, TestDocBasedVersionConstraints, TestConfig, TestSolrCoreParser, SynonymTokenizerTest, TestImpersonationWithHadoopAuth, TestRuleBasedAuthorizationPlugin, TestStressLucene, ClassificationUpdateProcessorTest, TestGraphTermsQParserPlugin, TestIBSimilarityFactory, SolrMetricReporterTest, TestFieldCacheReopen, TestDynamicFieldCollectionResource, CursorMarkTest, TestJsonRequest, PeerSyncTest, TestLegacyNumericRangeQueryBuilder, TestFreeTextSuggestions, TestWordDelimiterFilterFactory, TestCloudRecovery, SolrCoreTest, QueryElevationComponentTest, TestBackupRepositoryFactory, DistributedSuggestComponentTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestReplicationHandler, MetricUtilsTest, TestNumericTokenStream, TestBlobHandler, SpellingQueryConverterTest, TestRandomDVFaceting, SolrCloudReportersTest, SmileWriterTest, CollectionTooManyReplicasTest, FieldAnalysisRequestHandlerTest, TestPayloadCheckQParserPlugin, TestOmitPositions, BlockDirectoryTest, SpellCheckCollatorTest, TestSubQueryTransformer, TestSolr4Spatial, TestEmbeddedSolrServerConstructors, BasicDistributedZkTest, TestBadConfig, TestTolerantUpdateProcessorRandomCloud, DistributedExpandComponentTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionReuseTest, ReplicationFactorTest, TestJoin, MetricsConfigTest, TestConfigSetsAPI, TestDistributedSearch, TestCSVResponseWriter, SolrCLIZkUtilsTest, EchoParamsTest, ZkNodePropsTest, TestReplicaProperties, VersionInfoTest, ReplicaListTransformerTest, ConfigureRecoveryStrategyTest, TestInPlaceUpdatesDistrib, OutputWriterTest, OpenExchangeRatesOrgProviderTest, WrapperMergePolicyFactoryTest, TestMergePolicyConfig, TestSolrDeletionPolicy1, TestCustomSort, EnumFieldTest, TolerantUpdateProcessorTest, TestTrackingShardHandlerFactory, BadCopyFieldTest, TestPrepRecovery, LeaderElectionIntegrationTest, TestSizeLimitedDistributedMap, TestBinaryField, HdfsUnloadDistributedZkTest, TestSolrCoreSnapshots, TestSchemaVersionResource, SolrPluginUtilsTest, AddBlockUpdateTest, HdfsLockFactoryTest, HLLSerializationTest, StatsReloadRaceTest, ImplicitSnitchTest, DataDrivenBlockJoinTest, TestSchemaManager, TestLMDirichletSimilarityFactory, TestFunctionQuery, TestRandomFaceting, TestSchemaNameResource, SuggesterWFSTTest, BJQParserTest, TestBulkSchemaAPI, SSLMigrationTest, CoreMergeIndexesAdminHandlerTest, TestDownShardTolerantSearch, DistribDocExpirationUpdateProcessorTest, RankQueryTest, TestDistribIDF, RequiredFieldsTest, LoggingHandlerTest, TestCollationField, QueryParsingTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, TestComponentsName, HighlighterConfigTest, TestQuerySenderListener, UpdateParamsTest, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, CopyFieldTest, ResponseLogComponentTest, BadComponentTest, TestStressRecovery, DocValuesNotIndexedTest, HttpPartitionTest, MoveReplicaHDFSUlogDirTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, OverseerTaskQueueTest, RemoteQueryErrorTest, RestartWhileUpdatingTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [544/728 (2!)] on J0 in 211.15s, 1 test, 1 failure <<< FAILURES!
[...truncated 43909 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1354 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1354/
6 tests failed.
FAILED: org.apache.solr.TestDistributedSearch.test
Error Message:
Captured an uncaught exception in thread: Thread[id=153544, name=Thread-61588, state=RUNNABLE, group=TGRP-TestDistributedSearch]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=153544, name=Thread-61588, state=RUNNABLE, group=TGRP-TestDistributedSearch]
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F:DEF1D9552C1B0AE7]:0)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51740//collection1: java.lang.NullPointerException
at org.apache.solr.search.grouping.distributed.responseprocessor.StoredFieldsShardResponseProcessor.process(StoredFieldsShardResponseProcessor.java:41)
at org.apache.solr.handler.component.QueryComponent.handleGroupedResponses(QueryComponent.java:768)
at org.apache.solr.handler.component.QueryComponent.handleResponses(QueryComponent.java:751)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:423)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2473)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:378)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:322)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:252)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
at org.apache.solr.TestDistributedSearch$1.run(TestDistributedSearch.java:1145)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=159227, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=159253, name=zkCallback-25185-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=159228, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1124) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1110) at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:646) at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49) 4) Thread[id=159226, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 5) Thread[id=159252, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 6) Thread[id=159251, name=StoppableCommitThread-SendThread(127.0.0.1:50051), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 7) Thread[id=159281, name=zkCallback-25185-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=159280, name=zkCallback-25185-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=159227, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=159253, name=zkCallback-25185-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=159228, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1124)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1110)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:646)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
4) Thread[id=159226, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
5) Thread[id=159252, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
6) Thread[id=159251, name=StoppableCommitThread-SendThread(127.0.0.1:50051), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
7) Thread[id=159281, name=zkCallback-25185-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=159280, name=zkCallback-25185-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=159227, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.security.AccessController.doPrivileged(Native Method) at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114) at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99) at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.write(PrintStream.java:482) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) at java.io.PrintStream.write(PrintStream.java:527) at java.io.PrintStream.print(PrintStream.java:669) at java.io.PrintStream.println(PrintStream.java:823) at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748) at java.lang.Throwable.printStackTrace(Throwable.java:658) at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121) 2) Thread[id=159253, name=zkCallback-25185-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=159226, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 4) Thread[id=159251, name=StoppableCommitThread-SendThread(127.0.0.1:50051), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 5) Thread[id=159281, name=zkCallback-25185-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=159280, name=zkCallback-25185-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=159227, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.security.AccessController.doPrivileged(Native Method)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.write(PrintStream.java:482)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
at java.io.PrintStream.write(PrintStream.java:527)
at java.io.PrintStream.print(PrintStream.java:669)
at java.io.PrintStream.println(PrintStream.java:823)
at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
at java.lang.Throwable.printStackTrace(Throwable.java:658)
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
2) Thread[id=159253, name=zkCallback-25185-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=159226, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
4) Thread[id=159251, name=StoppableCommitThread-SendThread(127.0.0.1:50051), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
5) Thread[id=159281, name=zkCallback-25185-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=159280, name=zkCallback-25185-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F]:0)
FAILED: org.apache.solr.update.AutoCommitTest.testCommitWithin
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F:EC7789F701C9890A]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:878)
at org.apache.solr.update.AutoCommitTest.testCommitWithin(AutoCommitTest.java:353)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=id:529&qt=&start=0&rows=20&version=2.2
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:871)
... 40 more
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F:DEF1D9552C1B0AE7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F:DEF1D9552C1B0AE7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11425 lines...]
[junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/init-core-data-001
[junit4] 2> 206713 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=18 numCloses=18
[junit4] 2> 206713 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 206715 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 206715 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /f_cm/bq
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 206850 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 206861 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 206863 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 206878 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_36450_hdfs____.1vk5vu/webapp
[junit4] 2> 207403 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36450
[junit4] 2> 207586 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 207588 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 207603 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_49604_datanode____.3p3y4k/webapp
[junit4] 2> 208063 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:49604
[junit4] 2> 208107 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 208110 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 208124 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_39329_datanode____7av0b/webapp
[junit4] 2> 208398 INFO (IPC Server handler 8 on 57507) [ ] BlockStateChange BLOCK* processReport: from storage DS-44a596b8-d534-46ae-bd29-2d9c73543cb0 node DatanodeRegistration(127.0.0.1:37546, datanodeUuid=4513adc7-0e60-4ad6-9bac-f9e6cac96248, infoPort=33931, infoSecurePort=0, ipcPort=40018, storageInfo=lv=-56;cid=testClusterID;nsid=214005110;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 208398 INFO (IPC Server handler 8 on 57507) [ ] BlockStateChange BLOCK* processReport: from storage DS-c5b2b58f-a43f-433e-b18c-4b116a51d7f8 node DatanodeRegistration(127.0.0.1:37546, datanodeUuid=4513adc7-0e60-4ad6-9bac-f9e6cac96248, infoPort=33931, infoSecurePort=0, ipcPort=40018, storageInfo=lv=-56;cid=testClusterID;nsid=214005110;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 208585 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[56A5E68F82E7671F]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39329
[junit4] 2> 208817 INFO (IPC Server handler 2 on 57507) [ ] BlockStateChange BLOCK* processReport: from storage DS-8fb835dc-6540-471a-a00a-2d7c3d895fe9 node DatanodeRegistration(127.0.0.1:59499, datanodeUuid=9c13973d-ca97-4624-816e-012e2b5ac67b, infoPort=50084, infoSecurePort=0, ipcPort=39008, storageInfo=lv=-56;cid=testClusterID;nsid=214005110;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 208818 INFO (IPC Server handler 2 on 57507) [ ] BlockStateChange BLOCK* processReport: from storage DS-e92ec2be-7e58-41b1-9f02-1d5f64b85821 node DatanodeRegistration(127.0.0.1:59499, datanodeUuid=9c13973d-ca97-4624-816e-012e2b5ac67b, infoPort=50084, infoSecurePort=0, ipcPort=39008, storageInfo=lv=-56;cid=testClusterID;nsid=214005110;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 208864 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 208864 INFO (Thread-1560) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 208864 INFO (Thread-1560) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 208867 ERROR (Thread-1560) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 208964 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.ZkTestServer start zk server on port:40366
[junit4] 2> 208973 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15d40262f4c0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 208989 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 208995 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 208997 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 208998 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 209000 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 209001 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 209003 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 209004 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 209006 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 209008 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 209009 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 209012 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 209151 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 209153 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@74909458{/f_cm/bq,null,AVAILABLE}
[junit4] 2> 209153 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6d54f2df{HTTP/1.1,[http/1.1]}{127.0.0.1:56124}
[junit4] 2> 209154 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.Server Started @212169ms
[junit4] 2> 209154 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/f_cm/bq, hostPort=56124, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/control-001/cores}
[junit4] 2> 209154 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 209154 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 209155 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 209155 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 209155 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-14T08:10:30.894Z
[junit4] 2> 209159 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 209159 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/control-001/solr.xml
[junit4] 2> 209165 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 209168 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40366/solr
[junit4] 2> 209294 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 209299 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56124_f_cm%2Fbq
[junit4] 2> 209300 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.Overseer Overseer (id=98305299618463748-127.0.0.1:56124_f_cm%2Fbq-n_0000000000) starting
[junit4] 2> 209308 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56124_f_cm%2Fbq
[junit4] 2> 209313 INFO (OverseerStateUpdate-98305299618463748-127.0.0.1:56124_f_cm%2Fbq-n_0000000000) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 209396 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 209402 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 209402 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 209404 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/control-001/cores
[junit4] 2> 209428 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 209430 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40366/solr ready
[junit4] 2> 209431 INFO (qtp1774736340-3076) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56124_f_cm%252Fbq&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 209434 INFO (OverseerThreadFactory-1057-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 209542 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 209543 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 209650 INFO (zkCallback-165-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 210563 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 210578 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 210669 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 210694 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 210695 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 210695 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:57507/solr_hdfs_home
[junit4] 2> 210695 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 210695 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 210695 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:57507/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 210697 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57507/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 210706 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 210706 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 210706 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 210710 WARN (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 215712 WARN (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 220716 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 220726 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57507/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 220747 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57507/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 220755 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 220755 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 220760 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 220760 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@61b99753
[junit4] 2> 220811 INFO (IPC Server handler 8 on 57507) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37546 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-8fb835dc-6540-471a-a00a-2d7c3d895fe9:NORMAL:127.0.0.1:59499|RBW], ReplicaUC[[DISK]DS-44a596b8-d534-46ae-bd29-2d9c73543cb0:NORMAL:127.0.0.1:37546|FINALIZED]]} size 0
[junit4] 2> 220812 INFO (IPC Server handler 5 on 57507) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:59499 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44a596b8-d534-46ae-bd29-2d9c73543cb0:NORMAL:127.0.0.1:37546|FINALIZED], ReplicaUC[[DISK]DS-e92ec2be-7e58-41b1-9f02-1d5f64b85821:NORMAL:127.0.0.1:59499|FINALIZED]]} size 0
[junit4] 2> 220832 WARN (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 220904 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 220904 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 220904 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 220927 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 220928 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 220930 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=2.0458984375, floorSegmentMB=1.833984375, forceMergeDeletesPctAllowed=23.4980492491308, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8184596617865421
[junit4] 2> 220966 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@426e19f[control_collection_shard1_replica_n1] main]
[junit4] 2> 220967 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 220968 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 220969 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 220971 INFO (searcherExecutor-1060-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@426e19f[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 220971 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572884806589480960
[junit4] 2> 220978 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 220978 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 220978 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56124/f_cm/bq/control_collection_shard1_replica_n1/
[junit4] 2> 220978 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 220978 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56124/f_cm/bq/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 220978 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 220981 INFO (zkCallback-165-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 220984 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56124/f_cm/bq/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 221097 INFO (zkCallback-165-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 221136 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 221142 INFO (qtp1774736340-3078) [n:127.0.0.1:56124_f_cm%2Fbq c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=11600
[junit4] 2> 221150 INFO (qtp1774736340-3076) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 221245 INFO (zkCallback-165-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 221444 INFO (OverseerCollectionConfigSetProcessor-98305299618463748-127.0.0.1:56124_f_cm%2Fbq-n_0000000000) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 222150 INFO (qtp1774736340-3076) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56124_f_cm%252Fbq&wt=javabin&version=2} status=0 QTime=12719
[junit4] 2> 222175 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 222176 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40366/solr ready
[junit4] 2> 222176 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 222177 INFO (qtp1774736340-3074) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 222180 INFO (OverseerThreadFactory-1057-thread-2-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 222181 WARN (OverseerThreadFactory-1057-thread-2-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 222394 INFO (qtp1774736340-3074) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 222395 INFO (qtp1774736340-3074) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=217
[junit4] 2> 222539 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/shard-1-001 of type NRT
[junit4] 2> 222540 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 222541 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a4d3703{/f_cm/bq,null,AVAILABLE}
[junit4] 2> 222542 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64540729{HTTP/1.1,[http/1.1]}{127.0.0.1:38524}
[junit4] 2> 222543 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.e.j.s.Server Started @225557ms
[junit4] 2> 222543 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/f_cm/bq, hostPort=38524, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/shard-1-001/cores}
[junit4] 2> 222543 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 222543 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 222544 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 222544 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 222544 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-14T08:10:44.283Z
[junit4] 2> 222549 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 222554 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/shard-1-001/solr.xml
[junit4] 2> 222560 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 222565 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40366/solr
[junit4] 2> 222580 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 222583 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 222586 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38524_f_cm%2Fbq
[junit4] 2> 222587 INFO (zkCallback-165-thread-1-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 222591 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:38524_f_cm%2Fbq) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 222590 INFO (zkCallback-172-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 222690 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 222697 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 222697 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 222698 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[56A5E68F82E7671F]) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/shard-1-001/cores
[junit4] 2> 222725 INFO (qtp399968154-3128) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38524_f_cm%252Fbq&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 222728 INFO (OverseerCollectionConfigSetProcessor-98305299618463748-127.0.0.1:56124_f_cm%2Fbq-n_0000000000) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 222729 INFO (OverseerThreadFactory-1057-thread-3-processing-n:127.0.0.1:56124_f_cm%2Fbq) [n:127.0.0.1:56124_f_cm%2Fbq ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:38524_f_cm%2Fbq for creating new replica
[junit4] 2> 222731 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 222732 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 222836 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:38524_f_cm%2Fbq) [n:127.0.0.1:38524_f_cm%2Fbq ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 223754 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 223776 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 223868 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 223890 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 223890 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@530a7bc6
[junit4] 2> 223890 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:57507/solr_hdfs_home
[junit4] 2> 223891 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 223891 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 223891 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_56A5E68F82E7671F-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[hdfs://localhost:57507/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 223893 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57507/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 223902 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 223902 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 223906 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 223907 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57507/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 223925 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57507/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 223932 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 223932 INFO (qtp399968154-3130) [n:127.0.0.1:38524_f_cm%2Fbq c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s
[...truncated too long message...]
chronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=159280, name=zkCallback-25185-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=159227, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.security.AccessController.doPrivileged(Native Method)
[junit4] > at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
[junit4] > at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
[junit4] > at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
[junit4] > at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
[junit4] > at java.io.PrintStream.flush(PrintStream.java:338)
[junit4] > at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
[junit4] > at java.io.PrintStream.write(PrintStream.java:482)
[junit4] > at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
[junit4] > at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
[junit4] > at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
[junit4] > at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
[junit4] > at java.io.PrintStream.write(PrintStream.java:527)
[junit4] > at java.io.PrintStream.print(PrintStream.java:669)
[junit4] > at java.io.PrintStream.println(PrintStream.java:823)
[junit4] > at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
[junit4] > at java.lang.Throwable.printStackTrace(Throwable.java:658)
[junit4] > at java.lang.Throwable.printStackTrace(Throwable.java:643)
[junit4] > at java.lang.Throwable.printStackTrace(Throwable.java:634)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
[junit4] > 2) Thread[id=159253, name=zkCallback-25185-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=159226, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 4) Thread[id=159251, name=StoppableCommitThread-SendThread(127.0.0.1:50051), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > 5) Thread[id=159281, name=zkCallback-25185-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=159280, name=zkCallback-25185-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([56A5E68F82E7671F]:0)
[junit4] Completed [462/728 (4!)] on J0 in 87.59s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 44031 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1353 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1353/
4 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([35CE8052BB09C003:BD9ABF8815F5ADFB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest: 1) Thread[id=17124, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=17123, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:
1) Thread[id=17124, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=17123, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([35CE8052BB09C003]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=17124, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2) Thread[id=17123, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest] at java.io.PrintStream.println(PrintStream.java:805) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=17124, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
2) Thread[id=17123, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.io.PrintStream.println(PrintStream.java:805)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
at __randomizedtesting.SeedInfo.seed([35CE8052BB09C003]:0)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
Error Message:
expected:<[{indexVersion=1499932834675,generation=2,filelist=[_13k.cfe, _13k.cfs, _13k.si, _13l.fdt, _13l.fdx, _13l.fnm, _13l.nvd, _13l.nvm, _13l.si, _13l_Memory_0.ram, _13m.fdt, _13m.fdx, _13m.fnm, _13m.nvd, _13m.nvm, _13m.si, _13m_Memory_0.ram, _fo.fdt, _fo.fdx, _fo.fnm, _fo.nvd, _fo.nvm, _fo.si, _fo_Memory_0.ram, _v1.cfe, _v1.cfs, _v1.si, segments_2]}]> but was:<[{indexVersion=1499932834675,generation=2,filelist=[_13k.cfe, _13k.cfs, _13k.si, _13l.fdt, _13l.fdx, _13l.fnm, _13l.nvd, _13l.nvm, _13l.si, _13l_Memory_0.ram, _13m.fdt, _13m.fdx, _13m.fnm, _13m.nvd, _13m.nvm, _13m.si, _13m_Memory_0.ram, _fo.fdt, _fo.fdx, _fo.fnm, _fo.nvd, _fo.nvm, _fo.si, _fo_Memory_0.ram, _v1.cfe, _v1.cfs, _v1.si, segments_2]}, {indexVersion=1499932834675,generation=3,filelist=[_13n.cfe, _13n.cfs, _13n.si, _fo.fdt, _fo.fdx, _fo.fnm, _fo.nvd, _fo.nvm, _fo.si, _fo_Memory_0.ram, _v1.cfe, _v1.cfs, _v1.si, segments_3]}]>
Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1499932834675,generation=2,filelist=[_13k.cfe, _13k.cfs, _13k.si, _13l.fdt, _13l.fdx, _13l.fnm, _13l.nvd, _13l.nvm, _13l.si, _13l_Memory_0.ram, _13m.fdt, _13m.fdx, _13m.fnm, _13m.nvd, _13m.nvm, _13m.si, _13m_Memory_0.ram, _fo.fdt, _fo.fdx, _fo.fnm, _fo.nvd, _fo.nvm, _fo.si, _fo_Memory_0.ram, _v1.cfe, _v1.cfs, _v1.si, segments_2]}]> but was:<[{indexVersion=1499932834675,generation=2,filelist=[_13k.cfe, _13k.cfs, _13k.si, _13l.fdt, _13l.fdx, _13l.fnm, _13l.nvd, _13l.nvm, _13l.si, _13l_Memory_0.ram, _13m.fdt, _13m.fdx, _13m.fnm, _13m.nvd, _13m.nvm, _13m.si, _13m_Memory_0.ram, _fo.fdt, _fo.fdx, _fo.fnm, _fo.nvd, _fo.nvm, _fo.si, _fo_Memory_0.ram, _v1.cfe, _v1.cfs, _v1.si, segments_2]}, {indexVersion=1499932834675,generation=3,filelist=[_13n.cfe, _13n.cfs, _13n.si, _fo.fdt, _fo.fdx, _fo.fnm, _fo.nvd, _fo.nvm, _fo.si, _fo_Memory_0.ram, _v1.cfe, _v1.cfs, _v1.si, segments_3]}]>
at __randomizedtesting.SeedInfo.seed([35CE8052BB09C003:10199B62CB41CE00]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11960 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/init-core-data-001
[junit4] 2> 2504486 WARN (SUITE-ChaosMonkeySafeLeaderTest-seed#[35CE8052BB09C003]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
[junit4] 2> 2504487 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[35CE8052BB09C003]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2504500 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[35CE8052BB09C003]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2504500 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[35CE8052BB09C003]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2504502 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2504502 INFO (Thread-10065) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2504502 INFO (Thread-10065) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2504503 ERROR (Thread-10065) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2504605 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ZkTestServer start zk server on port:32847
[junit4] 2> 2504644 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2504645 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2504663 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2504664 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2504665 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2504665 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2504666 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2504679 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2504680 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2504680 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2504681 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2504682 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 2505188 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2505224 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2f15d3b{/,null,AVAILABLE}
[junit4] 2> 2505224 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@147b884d{HTTP/1.1,[http/1.1]}{127.0.0.1:45895}
[junit4] 2> 2505224 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server Started @2514681ms
[junit4] 2> 2505224 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=45895, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/control-001/cores}
[junit4] 2> 2505224 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2505225 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2505225 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2505225 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2505225 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-13T07:52:47.774Z
[junit4] 2> 2505237 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2505237 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/control-001/solr.xml
[junit4] 2> 2505262 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2505264 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:32847/solr
[junit4] 2> 2505801 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2505801 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45895_
[junit4] 2> 2505802 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.c.Overseer Overseer (id=98299567607119876-127.0.0.1:45895_-n_0000000000) starting
[junit4] 2> 2505876 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45895_
[junit4] 2> 2505877 INFO (zkCallback-1064-thread-1-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2506342 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2506345 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2506398 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2506399 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:45895_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/control-001/cores
[junit4] 2> 2506597 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2506670 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32847/solr ready
[junit4] 2> 2506707 INFO (qtp261395121-16743) [n:127.0.0.1:45895_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45895_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2506764 INFO (OverseerThreadFactory-2644-thread-1-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2507073 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2507073 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2507182 INFO (zkCallback-1064-thread-1-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2508161 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2508248 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2508681 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2508734 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2508735 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2508735 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2508735 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2508751 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=12.2314453125, floorSegmentMB=1.6884765625, forceMergeDeletesPctAllowed=25.40907569211492, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6246521026703976
[junit4] 2> 2508769 WARN (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2508999 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2508999 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2509017 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 2509017 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2509018 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2f29ebfa
[junit4] 2> 2509019 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a999ea[control_collection_shard1_replica_n1] main]
[junit4] 2> 2509041 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2509041 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2509042 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2509043 INFO (searcherExecutor-2647-thread-1-processing-n:127.0.0.1:45895_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6a999ea[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2509043 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572793086474452992
[junit4] 2> 2509063 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2509066 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2509066 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45895/control_collection_shard1_replica_n1/
[junit4] 2> 2509066 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2509066 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45895/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 2509066 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2509089 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45895/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 2509199 INFO (zkCallback-1064-thread-1-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2509251 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2509253 INFO (qtp261395121-16741) [n:127.0.0.1:45895_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2179
[junit4] 2> 2509289 INFO (qtp261395121-16743) [n:127.0.0.1:45895_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2509374 INFO (zkCallback-1064-thread-3-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2510304 INFO (qtp261395121-16743) [n:127.0.0.1:45895_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45895_&wt=javabin&version=2} status=0 QTime=3597
[junit4] 2> 2510308 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2510308 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32847/solr ready
[junit4] 2> 2510308 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2510309 INFO (qtp261395121-16744) [n:127.0.0.1:45895_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2510330 INFO (OverseerThreadFactory-2644-thread-2-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2510330 INFO (OverseerCollectionConfigSetProcessor-98299567607119876-127.0.0.1:45895_-n_0000000000) [n:127.0.0.1:45895_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2510330 WARN (OverseerThreadFactory-2644-thread-2-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 2510549 INFO (qtp261395121-16744) [n:127.0.0.1:45895_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2510549 INFO (qtp261395121-16744) [n:127.0.0.1:45895_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=239
[junit4] 2> 2510781 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-1-001 of type NRT
[junit4] 2> 2510782 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4cfb5ed1{/,null,AVAILABLE}
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a3c1de3{HTTP/1.1,[http/1.1]}{127.0.0.1:32782}
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server Started @2520240ms
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=32782, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-1-001/cores}
[junit4] 2> 2510783 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2510783 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2510784 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-13T07:52:53.333Z
[junit4] 2> 2510805 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2510805 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-1-001/solr.xml
[junit4] 2> 2510821 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2510824 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:32847/solr
[junit4] 2> 2510937 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2510938 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2510972 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32782_
[junit4] 2> 2510991 INFO (zkCallback-1064-thread-1-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2511097 INFO (zkCallback-1071-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2511113 INFO (zkCallback-1076-thread-1-processing-n:127.0.0.1:32782_) [n:127.0.0.1:32782_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2511592 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2511684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2511684 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2511718 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:32782_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-1-001/cores
[junit4] 2> 2511830 INFO (qtp261395121-16740) [n:127.0.0.1:45895_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:32782_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2511885 INFO (OverseerThreadFactory-2644-thread-3-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:32782_ for creating new replica
[junit4] 2> 2511885 INFO (OverseerCollectionConfigSetProcessor-98299567607119876-127.0.0.1:45895_-n_0000000000) [n:127.0.0.1:45895_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2511921 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2511922 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2512987 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2513037 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 2513418 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2513464 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 2513465 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2513465 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2513465 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-1-001/cores/collection1_shard2_replica_n0/data/]
[junit4] 2> 2513484 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=12.2314453125, floorSegmentMB=1.6884765625, forceMergeDeletesPctAllowed=25.40907569211492, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6246521026703976
[junit4] 2> 2513525 WARN (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2513717 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2513717 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2513718 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 2513718 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2513719 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@f71e9f6
[junit4] 2> 2513719 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@4a519021[collection1_shard2_replica_n0] main]
[junit4] 2> 2513720 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2513721 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2513721 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2513721 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572793091379691520
[junit4] 2> 2513734 INFO (searcherExecutor-2658-thread-1-processing-n:127.0.0.1:32782_ x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@4a519021[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2513748 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2513748 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2513748 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:32782/collection1_shard2_replica_n0/
[junit4] 2> 2513749 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2513749 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:32782/collection1_shard2_replica_n0/ has no replicas
[junit4] 2> 2513749 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2513763 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:32782/collection1_shard2_replica_n0/ shard2
[junit4] 2> 2513921 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2513922 INFO (qtp395845949-16786) [n:127.0.0.1:32782_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2000
[junit4] 2> 2513942 INFO (qtp261395121-16740) [n:127.0.0.1:45895_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:32782_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=2111
[junit4] 2> 2514626 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-2-001 of type NRT
[junit4] 2> 2514626 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2514627 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@b64150f{/,null,AVAILABLE}
[junit4] 2> 2514627 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5f28187d{HTTP/1.1,[http/1.1]}{127.0.0.1:41202}
[junit4] 2> 2514627 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server Started @2524084ms
[junit4] 2> 2514627 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=41202, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-2-001/cores}
[junit4] 2> 2514627 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2514628 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2514628 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2514628 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2514628 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-13T07:52:57.177Z
[junit4] 2> 2514746 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2514746 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-2-001/solr.xml
[junit4] 2> 2514773 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2514801 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:32847/solr
[junit4] 2> 2514969 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2514987 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2514988 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41202_
[junit4] 2> 2514989 INFO (zkCallback-1064-thread-1-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2514990 INFO (zkCallback-1071-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2514991 INFO (zkCallback-1082-thread-1-processing-n:127.0.0.1:41202_) [n:127.0.0.1:41202_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2514992 INFO (zkCallback-1076-thread-1-processing-n:127.0.0.1:32782_) [n:127.0.0.1:32782_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2515240 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2515256 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2515256 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2515257 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41202_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-2-001/cores
[junit4] 2> 2515343 INFO (qtp261395121-16745) [n:127.0.0.1:45895_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41202_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2515369 INFO (OverseerThreadFactory-2644-thread-4-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41202_ for creating new replica
[junit4] 2> 2515369 INFO (OverseerCollectionConfigSetProcessor-98299567607119876-127.0.0.1:45895_-n_0000000000) [n:127.0.0.1:45895_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2515371 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2515371 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2516541 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2516595 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema [collection1_shard1_replica_n0] Schema name=test
[junit4] 2> 2516876 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2516930 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 2516930 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n0' (registry 'solr.core.collection1.shard1.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2516930 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2516930 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [[collection1_shard1_replica_n0] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-2-001/cores/collection1_shard1_replica_n0], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-2-001/cores/collection1_shard1_replica_n0/data/]
[junit4] 2> 2516933 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=12.2314453125, floorSegmentMB=1.6884765625, forceMergeDeletesPctAllowed=25.40907569211492, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6246521026703976
[junit4] 2> 2516984 WARN (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2517242 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2517242 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2517243 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 2517243 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2517244 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1c84f64a
[junit4] 2> 2517245 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@49abdf5f[collection1_shard1_replica_n0] main]
[junit4] 2> 2517245 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2517246 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2517246 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2517246 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572793095075921920
[junit4] 2> 2517255 INFO (searcherExecutor-2669-thread-1-processing-n:127.0.0.1:41202_ x:collection1_shard1_replica_n0 s:shard1 c:collection1) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [collection1_shard1_replica_n0] Registered new searcher Searcher@49abdf5f[collection1_shard1_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2517296 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2517296 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2517296 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41202/collection1_shard1_replica_n0/
[junit4] 2> 2517296 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2517296 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:41202/collection1_shard1_replica_n0/ has no replicas
[junit4] 2> 2517296 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2517297 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41202/collection1_shard1_replica_n0/ shard1
[junit4] 2> 2517474 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2517475 INFO (qtp1077219453-16815) [n:127.0.0.1:41202_ c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2104
[junit4] 2> 2517477 INFO (qtp261395121-16745) [n:127.0.0.1:45895_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41202_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=2133
[junit4] 2> 2517906 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-3-001 of type NRT
[junit4] 2> 2517907 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2517908 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1123b1e1{/,null,AVAILABLE}
[junit4] 2> 2517920 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2c16164{HTTP/1.1,[http/1.1]}{127.0.0.1:41455}
[junit4] 2> 2517920 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.e.j.s.Server Started @2527377ms
[junit4] 2> 2517921 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=41455, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-3-001/cores}
[junit4] 2> 2517921 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2517921 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2517921 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2517921 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2517921 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-13T07:53:00.470Z
[junit4] 2> 2517940 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2517940 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-3-001/solr.xml
[junit4] 2> 2517958 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2517961 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:32847/solr
[junit4] 2> 2517991 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2517992 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2518005 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41455_
[junit4] 2> 2518007 INFO (zkCallback-1088-thread-1-processing-n:127.0.0.1:41455_) [n:127.0.0.1:41455_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2518007 INFO (zkCallback-1064-thread-3-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2518008 INFO (zkCallback-1082-thread-1-processing-n:127.0.0.1:41202_) [n:127.0.0.1:41202_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2518016 INFO (zkCallback-1071-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2518050 INFO (zkCallback-1076-thread-1-processing-n:127.0.0.1:32782_) [n:127.0.0.1:32782_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2518321 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2518337 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2518337 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2518355 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[35CE8052BB09C003]) [n:127.0.0.1:41455_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-3-001/cores
[junit4] 2> 2518497 INFO (qtp539640805-16844) [n:127.0.0.1:41455_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41455_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2518541 INFO (OverseerThreadFactory-2644-thread-5-processing-n:127.0.0.1:45895_) [n:127.0.0.1:45895_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41455_ for creating new replica
[junit4] 2> 2518541 INFO (OverseerCollectionConfigSetProcessor-98299567607119876-127.0.0.1:45895_-n_0000000000) [n:127.0.0.1:45895_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2518554 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2518555 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2519620 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2519701 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema [collection1_shard2_replica_n1] Schema name=test
[junit4] 2> 2520147 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2520185 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n1' using configuration from collection collection1, trusted=true
[junit4] 2> 2520186 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n1' (registry 'solr.core.collection1.shard2.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7b043e21
[junit4] 2> 2520186 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2520186 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [[collection1_shard2_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-3-001/cores/collection1_shard2_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_35CE8052BB09C003-001/shard-3-001/cores/collection1_shard2_replica_n1/data/]
[junit4] 2> 2520201 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=12.2314453125, floorSegmentMB=1.6884765625, forceMergeDeletesPctAllowed=25.40907569211492, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6246521026703976
[junit4] 2> 2520246 WARN (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2520385 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2520385 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2520398 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 2520398 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2520399 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6a7e42ab
[junit4] 2> 2520399 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7309faf2[collection1_shard2_replica_n1] main]
[junit4] 2> 2520400 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2520401 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2520401 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2520402 INFO (searcherExecutor-2680-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.SolrCore [collection1_shard2_replica_n1] Registered new searcher Searcher@7309faf2[collection1_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2520402 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572793098385227776
[junit4] 2> 2520417 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_n1
[junit4] 2> 2520435 INFO (updateExecutor-1085-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2520435 INFO (qtp539640805-16842) [n:127.0.0.1:41455_ c:collection1 s:shard2 x:collection1_shard2_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n1&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1881
[junit4] 2> 2520436 INFO (recoveryExecutor-1086-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2520438 INFO (recoveryExecutor-1086-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2520438 INFO (recoveryExecutor-1086-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_n1]
[junit4] 2> 2520438 INFO (recoveryExecutor-1086-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2520438 INFO (recoveryExecutor-1086-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_n1] as recovering, leader is [http://127.0.0.1:32782/collection1_shard2_replica_n0/] and I am [http://127.0.0.1:41455/collection1_shard2_replica_n1/]
[junit4] 2> 2520440 INFO (qtp539640805-16844) [n:127.0.0.1:41455_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41455_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1942
[junit4] 2> 2520483 INFO (recoveryExecutor-1086-thread-1-processing-n:127.0.0.1:41455_ x:collection1_shard2_replica_n1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:41455_ c:collection1 s:shard2 r:core_node3 x:collection1_shard2_replica_n1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:32782]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_n0&nodeName=127.0.0.1:41455_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2520484 INFO (qtp395845949-16784) [n:127.0.0.1:32782_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit
[...truncated too long message...]
ection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_2.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189483 INFO (qtp1023939529-67357) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_2.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189483 INFO (qtp1023939529-67358) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_2.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189484 INFO (qtp1023939529-67359) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_2.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189485 INFO (qtp1023939529-67352) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_2_Memory_0.ram&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189485 INFO (qtp1023939529-67355) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_2.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189486 INFO (qtp1023939529-67353) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 3189486 INFO (indexFetcher-3748-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=12892) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@393c309e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65905336)
[junit4] 2> 3189487 INFO (indexFetcher-3748-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 3189487 INFO (indexFetcher-3748-thread-1) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 3189488 INFO (indexFetcher-3748-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c13b5d0[collection1] main]
[junit4] 2> 3189489 INFO (searcherExecutor-3744-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7c13b5d0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C210) Uninverting(_1(8.0.0):C210) Uninverting(_2(8.0.0):C73)))}
[junit4] 2> 3189580 INFO (qtp1345234278-67374) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=493 status=0 QTime=0
[junit4] 2> 3189580 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.h.TestReplicationHandler Waited for 700ms and found 493 docs
[junit4] 2> 3189580 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.SolrTestCaseJ4 ###Ending doTestReplicateAfterStartup
[junit4] 2> 3189581 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@45af9ffd{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3189581 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=776399066
[junit4] 2> 3189581 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3189582 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3189584 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3189584 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3189584 INFO (coreCloseExecutor-3750-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@6457c140
[junit4] 2> 3189585 INFO (coreCloseExecutor-3750-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1683472704
[junit4] 2> 3189587 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@65f451f9{/solr,null,UNAVAILABLE}
[junit4] 2> 3189588 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@15843ec0{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3189588 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=984435192
[junit4] 2> 3189588 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3189588 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3189588 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3189588 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3189589 INFO (coreCloseExecutor-3752-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@6499cafd
[junit4] 2> 3189589 INFO (coreCloseExecutor-3752-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1687800573
[junit4] 2> 3189590 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[35CE8052BB09C003]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@575b1dd5{/solr,null,UNAVAILABLE}
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_35CE8052BB09C003-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {name=PostingsFormat(name=Memory), id=PostingsFormat(name=Memory), newname=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=362, maxMBSortInHeap=7.6174316347626005, sim=RandomSimilarity(queryNorm=false): {}, locale=uk, timezone=America/Eirunepe
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=180202200,total=529006592
[junit4] 2> NOTE: All tests run in this JVM: [SolrCloudReportersTest, NodeMutatorTest, TestBlobHandler, TestCollationFieldDocValues, SpellingQueryConverterTest, TestOmitPositions, TestDocBasedVersionConstraints, TestPerFieldSimilarityWithDefaultOverride, TestFieldCacheReopen, OutputWriterTest, TestWriterPerf, TestFaceting, SolrCoreTest, DefaultValueUpdateProcessorTest, TestIBSimilarityFactory, TestSolrCLIRunExample, TestSchemaNameResource, MoveReplicaHDFSUlogDirTest, UUIDFieldTest, TestAnalyzeInfixSuggestions, DistributedVersionInfoTest, TestRandomCollapseQParserPlugin, TestComponentsName, TestFieldResource, DebugComponentTest, DistributedSuggestComponentTest, TestManagedSchemaAPI, CursorMarkTest, TestDistributedSearch, DistributedSpellCheckComponentTest, TestSolr4Spatial, TestConfigsApi, SolrGraphiteReporterTest, TestCustomSort, HdfsUnloadDistributedZkTest, TestManagedStopFilterFactory, TestHighlightDedupGrouping, BlobRepositoryCloudTest, SolrIndexSplitterTest, BlockDirectoryTest, TestSchemaVersionResource, BitVectorTest, SolrRequestParserTest, TestSolrConfigHandlerConcurrent, CloudExitableDirectoryReaderTest, TestSolrQueryParser, TestDynamicFieldCollectionResource, TestCloudJSONFacetJoinDomain, TestInfoStreamLogging, TestLRUStatsCache, TestCollationField, BadCopyFieldTest, SuggesterWFSTTest, TestPHPSerializedResponseWriter, TestSQLHandler, TestConfigSetProperties, ImplicitSnitchTest, SSLMigrationTest, TestCollapseQParserPlugin, OverseerCollectionConfigSetProcessorTest, TestReplicaProperties, TestIndexingPerformance, TestSolrCoreParser, SolrCLIZkUtilsTest, EchoParamsTest, TestJsonRequest, TestRuleBasedAuthorizationPlugin, TestWordDelimiterFilterFactory, TestBulkSchemaAPI, HdfsBasicDistributedZk2Test, DistributedFacetPivotLongTailTest, TestManagedResource, BJQParserTest, DeleteShardTest, TestUseDocValuesAsStored2, TestJavabinTupleStreamParser, TestCloudManagedSchema, CdcrBootstrapTest, TestReplicationHandler]
[junit4] Completed [647/728 (2!)] on J0 in 480.77s, 16 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 43561 lines...]
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 1352 - Still
Failing
Posted by Steve Rowe <sa...@gmail.com>.
The disk allocation for the “lucene2” node was increased and it’s back in rotation now after being offline for a while to perform the operation. <https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/3/> is in progress now.
--
Steve
www.lucidworks.com
> On Jul 12, 2017, at 9:26 AM, Steve Rowe <sa...@gmail.com> wrote:
>
> I created <https://issues.apache.org/jira/browse/INFRA-14572> for the "lucene2” disk space problem.
>
> On Hipchat Chris Thistlthwaite (@christ) said he would start looking at the box.
>
> --
> Steve
> www.lucidworks.com
>
>> On Jul 12, 2017, at 8:54 AM, Steve Rowe <sa...@gmail.com> wrote:
>>
>> Looks like the new lucene2 node has run out of disk space. I’ll go ask about it on Infra’s hipchat channel.
>>
>> --
>> Steve
>> www.lucidworks.com
>>
>>> On Jul 12, 2017, at 3:11 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
>>>
>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1352/
>>>
>>> 1 tests failed.
>>> FAILED: TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml.[empty]
>>>
>>> Error Message:
>>>
>>>
>>> Stack Trace:
>>> Test report file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml was length 0
>>>
>>>
>>>
>>> Build Log:
>>> [...truncated 262 lines...]
>>> [junit4] JVM J0: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163200394456052706494.syserr
>>> [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
>>> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
>>> [junit4] at java.security.AccessController.doPrivileged(Native Method)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
>>> [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
>>> [junit4] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
>>> [junit4] at java.io.PrintStream.flush(PrintStream.java:338)
>>> [junit4] at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
>>> [junit4] at java.io.PrintStream.write(PrintStream.java:482)
>>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
>>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>>> [junit4] at java.io.Writer.write(Writer.java:157)
>>> [junit4] at java.util.logging.StreamHandler.publish(StreamHandler.java:224)
>>> [junit4] at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:116)
>>> [junit4] at java.util.logging.Logger.log(Logger.java:738)
>>> [junit4] at java.util.logging.Logger.doLog(Logger.java:765)
>>> [junit4] at java.util.logging.Logger.log(Logger.java:876)
>>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException(RandomizedRunner.java:531)
>>> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1057)
>>> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
>>> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
>>> [junit4] at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:28)
>>> [junit4] at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>>> [junit4] Caused by: java.io.IOException: No space left on device
>>> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
>>> [junit4] at java.io.FileOut
>>> [junit4] putStream.write(FileOutputStream.java:326)
>>> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
>>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
>>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
>>> [junit4] ... 26 more
>>> [junit4] <<< JVM J0: EOF ----
>>>
>>> [...truncated 4 lines...]
>>> [junit4] JVM J1: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6343353887351313582581.syserr
>>> [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
>>> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
>>> [junit4] at java.security.AccessController.doPrivileged(Native Method)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testRunStarted(RunListenerEmitter.java:40)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testRunStarted(NoExceptionRunListenerDecorator.java:23)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testRunStarted(BeforeAfterRunListenerDecorator.java:24)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$1.notifyListener(OrderedRunNotifier.java:82)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:49)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestRunStarted(OrderedRunNotifier.java:79)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:249)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
>>> [junit4] Caused by: java.io.IOException: No space left on device
>>> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
>>> [junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
>>> [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
>>> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:121)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
>>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>>> [junit4] at sun.nio.cs.Strea
>>> [junit4] mEncoder.write(StreamEncoder.java:135)
>>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:24)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:31)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeDescription(AbstractEvent.java:119)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEventWithDescription.serialize(AbstractEventWithDescription.java:29)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.SuiteStartedEvent.serialize(SuiteStartedEvent.java:36)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
>>> [junit4] ... 13 more
>>> [junit4] <<< JVM J1: EOF ----
>>>
>>> [junit4] JVM J2: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345429632897914706142.syserr
>>> [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
>>> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
>>> [junit4] at java.security.AccessController.doPrivileged(Native Method)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testFailure(RunListenerEmitter.java:54)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testFailure(NoExceptionRunListenerDecorator.java:55)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testFailure(BeforeAfterRunListenerDecorator.java:60)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$4.notifyListener(OrderedRunNotifier.java:129)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestFailure(OrderedRunNotifier.java:126)
>>> [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$SubNotifier.fireTestFailure(ThreadLeakControl.java:184)
>>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.fireTestFailure(RandomizedRunner.java:834)
>>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:922)
>>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
>>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
>>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>>> [junit4] at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>> [junit4] at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverr
>>> [junit4] idesOnMethodsRule.java:40)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>> [junit4] at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>>> [junit4] at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>>> [junit4] at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>>> [junit4] at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>> [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>>> [junit4] at java.lang.Thread.run(Thread.java:748)
>>> [junit4] Caused by: java.io.IOException: No space left on device
>>> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
>>> [junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
>>> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
>>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
>>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:566)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.FailureEvent.serialize(FailureEvent.java:46)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4
>>> [junit4] .events.Serializer$2.run(Serializer.java:119)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
>>> [junit4] ... 30 more
>>> [junit4] <<< JVM J2: EOF ----
>>>
>>> [...truncated 2 lines...]
>>> [junit4] ERROR: JVM J0 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0 -Djunit4.childvm.id=0 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163348084582467318555.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6166850120803246205.suites -stdin
>>> [junit4] ERROR: JVM J0 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
>>> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>>> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>> [junit4] at java.lang.Thread.run(Thread.java:748)
>>> [junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6341196093693912703591.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6348978994367824159899.suites -stdin
>>> [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
>>> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>>> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>> [junit4] at java.lang.Thread.run(Thread.java:748)
>>> [junit4] ERROR: JVM J2 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2 -Djunit4.childvm.id=2 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6343973825201621029068.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345414770058120710840.suites -stdin
>>> [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
>>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
>>> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>>> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>> [junit4] at java.lang.Thread.run(Thread.java:748)
>>>
>>> BUILD FAILED
>>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
>>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:754: The following error occurred while executing this line:
>>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
>>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build.xml:50: The following error occurred while executing this line:
>>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1455: The following error occurred while executing this line:
>>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:985: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>>>
>>> Total time: 2 minutes 1 second
>>> Build step 'Invoke Ant' marked build as failure
>>> Archiving artifacts
>>> No artifacts from Lucene-Solr-NightlyTests-master #1349 to compare, so performing full copy of artifacts
>>> Recording test results
>>> Email was triggered for: Failure - Any
>>> Sending email for trigger: Failure - Any
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 1352 - Still
Failing
Posted by Steve Rowe <sa...@gmail.com>.
I created <https://issues.apache.org/jira/browse/INFRA-14572> for the "lucene2” disk space problem.
On Hipchat Chris Thistlthwaite (@christ) said he would start looking at the box.
--
Steve
www.lucidworks.com
> On Jul 12, 2017, at 8:54 AM, Steve Rowe <sa...@gmail.com> wrote:
>
> Looks like the new lucene2 node has run out of disk space. I’ll go ask about it on Infra’s hipchat channel.
>
> --
> Steve
> www.lucidworks.com
>
>> On Jul 12, 2017, at 3:11 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
>>
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1352/
>>
>> 1 tests failed.
>> FAILED: TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml.[empty]
>>
>> Error Message:
>>
>>
>> Stack Trace:
>> Test report file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml was length 0
>>
>>
>>
>> Build Log:
>> [...truncated 262 lines...]
>> [junit4] JVM J0: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163200394456052706494.syserr
>> [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
>> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
>> [junit4] at java.security.AccessController.doPrivileged(Native Method)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
>> [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
>> [junit4] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
>> [junit4] at java.io.PrintStream.flush(PrintStream.java:338)
>> [junit4] at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
>> [junit4] at java.io.PrintStream.write(PrintStream.java:482)
>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>> [junit4] at java.io.Writer.write(Writer.java:157)
>> [junit4] at java.util.logging.StreamHandler.publish(StreamHandler.java:224)
>> [junit4] at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:116)
>> [junit4] at java.util.logging.Logger.log(Logger.java:738)
>> [junit4] at java.util.logging.Logger.doLog(Logger.java:765)
>> [junit4] at java.util.logging.Logger.log(Logger.java:876)
>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException(RandomizedRunner.java:531)
>> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1057)
>> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
>> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
>> [junit4] at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:28)
>> [junit4] at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>> [junit4] Caused by: java.io.IOException: No space left on device
>> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
>> [junit4] at java.io.FileOut
>> [junit4] putStream.write(FileOutputStream.java:326)
>> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
>> [junit4] ... 26 more
>> [junit4] <<< JVM J0: EOF ----
>>
>> [...truncated 4 lines...]
>> [junit4] JVM J1: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6343353887351313582581.syserr
>> [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
>> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
>> [junit4] at java.security.AccessController.doPrivileged(Native Method)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testRunStarted(RunListenerEmitter.java:40)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testRunStarted(NoExceptionRunListenerDecorator.java:23)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testRunStarted(BeforeAfterRunListenerDecorator.java:24)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$1.notifyListener(OrderedRunNotifier.java:82)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:49)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestRunStarted(OrderedRunNotifier.java:79)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:249)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
>> [junit4] Caused by: java.io.IOException: No space left on device
>> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
>> [junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
>> [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
>> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:121)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>> [junit4] at sun.nio.cs.Strea
>> [junit4] mEncoder.write(StreamEncoder.java:135)
>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:24)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:31)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeDescription(AbstractEvent.java:119)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEventWithDescription.serialize(AbstractEventWithDescription.java:29)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.SuiteStartedEvent.serialize(SuiteStartedEvent.java:36)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
>> [junit4] ... 13 more
>> [junit4] <<< JVM J1: EOF ----
>>
>> [junit4] JVM J2: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345429632897914706142.syserr
>> [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
>> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
>> [junit4] at java.security.AccessController.doPrivileged(Native Method)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testFailure(RunListenerEmitter.java:54)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testFailure(NoExceptionRunListenerDecorator.java:55)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testFailure(BeforeAfterRunListenerDecorator.java:60)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$4.notifyListener(OrderedRunNotifier.java:129)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestFailure(OrderedRunNotifier.java:126)
>> [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$SubNotifier.fireTestFailure(ThreadLeakControl.java:184)
>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.fireTestFailure(RandomizedRunner.java:834)
>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:922)
>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
>> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>> [junit4] at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> [junit4] at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverr
>> [junit4] idesOnMethodsRule.java:40)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> [junit4] at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>> [junit4] at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
>> [junit4] at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>> [junit4] at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
>> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
>> [junit4] at java.lang.Thread.run(Thread.java:748)
>> [junit4] Caused by: java.io.IOException: No space left on device
>> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
>> [junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
>> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
>> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
>> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
>> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
>> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:566)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.FailureEvent.serialize(FailureEvent.java:46)
>> [junit4] at com.carrotsearch.ant.tasks.junit4
>> [junit4] .events.Serializer$2.run(Serializer.java:119)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
>> [junit4] ... 30 more
>> [junit4] <<< JVM J2: EOF ----
>>
>> [...truncated 2 lines...]
>> [junit4] ERROR: JVM J0 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0 -Djunit4.childvm.id=0 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163348084582467318555.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6166850120803246205.suites -stdin
>> [junit4] ERROR: JVM J0 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
>> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> [junit4] at java.lang.Thread.run(Thread.java:748)
>> [junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6341196093693912703591.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6348978994367824159899.suites -stdin
>> [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
>> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> [junit4] at java.lang.Thread.run(Thread.java:748)
>> [junit4] ERROR: JVM J2 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2 -Djunit4.childvm.id=2 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6343973825201621029068.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345414770058120710840.suites -stdin
>> [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
>> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
>> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> [junit4] at java.lang.Thread.run(Thread.java:748)
>>
>> BUILD FAILED
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:754: The following error occurred while executing this line:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build.xml:50: The following error occurred while executing this line:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1455: The following error occurred while executing this line:
>> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:985: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>>
>> Total time: 2 minutes 1 second
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> No artifacts from Lucene-Solr-NightlyTests-master #1349 to compare, so performing full copy of artifacts
>> Recording test results
>> Email was triggered for: Failure - Any
>> Sending email for trigger: Failure - Any
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 1352 - Still
Failing
Posted by Steve Rowe <sa...@gmail.com>.
Looks like the new lucene2 node has run out of disk space. I’ll go ask about it on Infra’s hipchat channel.
--
Steve
www.lucidworks.com
> On Jul 12, 2017, at 3:11 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
>
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1352/
>
> 1 tests failed.
> FAILED: TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml.[empty]
>
> Error Message:
>
>
> Stack Trace:
> Test report file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml was length 0
>
>
>
> Build Log:
> [...truncated 262 lines...]
> [junit4] JVM J0: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163200394456052706494.syserr
> [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
> [junit4] at java.security.AccessController.doPrivileged(Native Method)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
> [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
> [junit4] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
> [junit4] at java.io.PrintStream.flush(PrintStream.java:338)
> [junit4] at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
> [junit4] at java.io.PrintStream.write(PrintStream.java:482)
> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
> [junit4] at java.io.Writer.write(Writer.java:157)
> [junit4] at java.util.logging.StreamHandler.publish(StreamHandler.java:224)
> [junit4] at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:116)
> [junit4] at java.util.logging.Logger.log(Logger.java:738)
> [junit4] at java.util.logging.Logger.doLog(Logger.java:765)
> [junit4] at java.util.logging.Logger.log(Logger.java:876)
> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException(RandomizedRunner.java:531)
> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1057)
> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
> [junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
> [junit4] at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:28)
> [junit4] at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
> [junit4] Caused by: java.io.IOException: No space left on device
> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
> [junit4] at java.io.FileOut
> [junit4] putStream.write(FileOutputStream.java:326)
> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
> [junit4] ... 26 more
> [junit4] <<< JVM J0: EOF ----
>
> [...truncated 4 lines...]
> [junit4] JVM J1: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6343353887351313582581.syserr
> [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
> [junit4] at java.security.AccessController.doPrivileged(Native Method)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testRunStarted(RunListenerEmitter.java:40)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testRunStarted(NoExceptionRunListenerDecorator.java:23)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testRunStarted(BeforeAfterRunListenerDecorator.java:24)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$1.notifyListener(OrderedRunNotifier.java:82)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:49)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestRunStarted(OrderedRunNotifier.java:79)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:249)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
> [junit4] Caused by: java.io.IOException: No space left on device
> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
> [junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
> [junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:121)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
> [junit4] at sun.nio.cs.Strea
> [junit4] mEncoder.write(StreamEncoder.java:135)
> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:24)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:31)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeDescription(AbstractEvent.java:119)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEventWithDescription.serialize(AbstractEventWithDescription.java:29)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.SuiteStartedEvent.serialize(SuiteStartedEvent.java:36)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
> [junit4] ... 13 more
> [junit4] <<< JVM J1: EOF ----
>
> [junit4] JVM J2: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345429632897914706142.syserr
> [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
> [junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
> [junit4] at java.security.AccessController.doPrivileged(Native Method)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testFailure(RunListenerEmitter.java:54)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testFailure(NoExceptionRunListenerDecorator.java:55)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testFailure(BeforeAfterRunListenerDecorator.java:60)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$4.notifyListener(OrderedRunNotifier.java:129)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestFailure(OrderedRunNotifier.java:126)
> [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$SubNotifier.fireTestFailure(ThreadLeakControl.java:184)
> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.fireTestFailure(RandomizedRunner.java:834)
> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:922)
> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
> [junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
> [junit4] at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> [junit4] at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> [junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverr
> [junit4] idesOnMethodsRule.java:40)
> [junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> [junit4] at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> [junit4] at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> [junit4] at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> [junit4] at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> [junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> [junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
> [junit4] at java.lang.Thread.run(Thread.java:748)
> [junit4] Caused by: java.io.IOException: No space left on device
> [junit4] at java.io.FileOutputStream.writeBytes(Native Method)
> [junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
> [junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
> [junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
> [junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
> [junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
> [junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
> [junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:566)
> [junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.FailureEvent.serialize(FailureEvent.java:46)
> [junit4] at com.carrotsearch.ant.tasks.junit4
> [junit4] .events.Serializer$2.run(Serializer.java:119)
> [junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
> [junit4] ... 30 more
> [junit4] <<< JVM J2: EOF ----
>
> [...truncated 2 lines...]
> [junit4] ERROR: JVM J0 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0 -Djunit4.childvm.id=0 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163348084582467318555.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6166850120803246205.suites -stdin
> [junit4] ERROR: JVM J0 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [junit4] at java.lang.Thread.run(Thread.java:748)
> [junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6341196093693912703591.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6348978994367824159899.suites -stdin
> [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [junit4] at java.lang.Thread.run(Thread.java:748)
> [junit4] ERROR: JVM J2 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2 -Djunit4.childvm.id=2 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6343973825201621029068.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345414770058120710840.suites -stdin
> [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
> [junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
> [junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> [junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [junit4] at java.lang.Thread.run(Thread.java:748)
>
> BUILD FAILED
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:754: The following error occurred while executing this line:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build.xml:50: The following error occurred while executing this line:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1455: The following error occurred while executing this line:
> /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:985: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
>
> Total time: 2 minutes 1 second
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> No artifacts from Lucene-Solr-NightlyTests-master #1349 to compare, so performing full copy of artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1352 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1352/
1 tests failed.
FAILED: TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml.[empty]
Error Message:
Stack Trace:
Test report file /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/TEST-org.apache.lucene.search.TestSimpleExplanationsWithFillerDocs.xml was length 0
Build Log:
[...truncated 262 lines...]
[junit4] JVM J0: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163200394456052706494.syserr
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
[junit4] at java.security.AccessController.doPrivileged(Native Method)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
[junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
[junit4] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
[junit4] at java.io.PrintStream.flush(PrintStream.java:338)
[junit4] at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
[junit4] at java.io.PrintStream.write(PrintStream.java:482)
[junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
[junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
[junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
[junit4] at java.io.Writer.write(Writer.java:157)
[junit4] at java.util.logging.StreamHandler.publish(StreamHandler.java:224)
[junit4] at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:116)
[junit4] at java.util.logging.Logger.log(Logger.java:738)
[junit4] at java.util.logging.Logger.doLog(Logger.java:765)
[junit4] at java.util.logging.Logger.log(Logger.java:876)
[junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException(RandomizedRunner.java:531)
[junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1057)
[junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
[junit4] at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
[junit4] at com.carrotsearch.randomizedtesting.RunnerThreadGroup.uncaughtException(RunnerThreadGroup.java:28)
[junit4] at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
[junit4] Caused by: java.io.IOException: No space left on device
[junit4] at java.io.FileOutputStream.writeBytes(Native Method)
[junit4] at java.io.FileOut
[junit4] putStream.write(FileOutputStream.java:326)
[junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
[junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
[junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
[junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
[junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
[junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
[junit4] ... 26 more
[junit4] <<< JVM J0: EOF ----
[...truncated 4 lines...]
[junit4] JVM J1: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6343353887351313582581.syserr
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
[junit4] at java.security.AccessController.doPrivileged(Native Method)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testRunStarted(RunListenerEmitter.java:40)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testRunStarted(NoExceptionRunListenerDecorator.java:23)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testRunStarted(BeforeAfterRunListenerDecorator.java:24)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$1.notifyListener(OrderedRunNotifier.java:82)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:49)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestRunStarted(OrderedRunNotifier.java:79)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:249)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] Caused by: java.io.IOException: No space left on device
[junit4] at java.io.FileOutputStream.writeBytes(Native Method)
[junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
[junit4] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
[junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:121)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
[junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
[junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
[junit4] at sun.nio.cs.Strea
[junit4] mEncoder.write(StreamEncoder.java:135)
[junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
[junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:572)
[junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:24)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.JsonHelpers.writeDescription(JsonHelpers.java:31)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeDescription(AbstractEvent.java:119)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.AbstractEventWithDescription.serialize(AbstractEventWithDescription.java:29)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.SuiteStartedEvent.serialize(SuiteStartedEvent.java:36)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
[junit4] ... 13 more
[junit4] <<< JVM J1: EOF ----
[junit4] JVM J2: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345429632897914706142.syserr
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] WARN: Unhandled exception in event serialization. -> java.security.PrivilegedActionException: java.io.IOException: No space left on device
[junit4] at java.security.AccessController.doPrivileged(Native Method)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.RunListenerEmitter.testFailure(RunListenerEmitter.java:54)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.NoExceptionRunListenerDecorator.testFailure(NoExceptionRunListenerDecorator.java:55)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.BeforeAfterRunListenerDecorator.testFailure(BeforeAfterRunListenerDecorator.java:60)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$4.notifyListener(OrderedRunNotifier.java:129)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier$SafeNotifier.run(OrderedRunNotifier.java:63)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.OrderedRunNotifier.fireTestFailure(OrderedRunNotifier.java:126)
[junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$SubNotifier.fireTestFailure(ThreadLeakControl.java:184)
[junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.fireTestFailure(RandomizedRunner.java:834)
[junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:922)
[junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverr
[junit4] idesOnMethodsRule.java:40)
[junit4] at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4] Caused by: java.io.IOException: No space left on device
[junit4] at java.io.FileOutputStream.writeBytes(Native Method)
[junit4] at java.io.FileOutputStream.write(FileOutputStream.java:326)
[junit4] at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122)
[junit4] at com.carrotsearch.ant.tasks.junit4.slave.EventsOutputStream.write(EventsOutputStream.java:28)
[junit4] at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
[junit4] at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:282)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
[junit4] at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:135)
[junit4] at java.io.OutputStreamWriter.write(OutputStreamWriter.java:220)
[junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.string(JsonWriter.java:566)
[junit4] at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.value(JsonWriter.java:415)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.FailureEvent.serialize(FailureEvent.java:46)
[junit4] at com.carrotsearch.ant.tasks.junit4
[junit4] .events.Serializer$2.run(Serializer.java:119)
[junit4] at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
[junit4] ... 30 more
[junit4] <<< JVM J2: EOF ----
[...truncated 2 lines...]
[junit4] ERROR: JVM J0 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J0 -Djunit4.childvm.id=0 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6163348084582467318555.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J0-20170712_070941_6166850120803246205.suites -stdin
[junit4] ERROR: JVM J0 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6341196093693912703591.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J1-20170712_070941_6348978994367824159899.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:748)
[junit4] ERROR: JVM J2 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_131/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=55E4911CB74D62C9 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.luceneMatchVersion=8.0.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/logging.properties -Dtests.nightly=true -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp -Dcommon.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene -Dclover.db.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/clover/db -Djava.security.policy=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=8.0.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout -Djunit4.childvm.cwd=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2 -Djunit4.childvm.id=2 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/codecs/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/test-framework/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/junit-4.10.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/test-framework/lib/randomizedtesting-runner-2.5.2.jar:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/java:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/home/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/usr/local/asfpackages/java/jdk1.8.0_131/lib/tools.jar:/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6343973825201621029068.events @/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/temp/junit4-J2-20170712_070941_6345414770058120710840.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1540)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:754: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build.xml:50: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1455: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:985: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
Total time: 2 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No artifacts from Lucene-Solr-NightlyTests-master #1349 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1351 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1351/
No tests ran.
Build Log:
[...truncated 79 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:422: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:459: Ivy is not available
Total time: 0 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: No test report files were found. Configuration error?
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1350 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1350/
9 tests failed.
FAILED: org.apache.lucene.codecs.lucene70.TestIndexedDISI.testRandom
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
FAILED: org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
FAILED: org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit
Error Message:
this IndexWriter is closed
Stack Trace:
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
at org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit(TestIndexingSequenceNumbers.java:230)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
FAILED: org.apache.lucene.search.TestDoubleRangeFieldQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
FAILED: org.apache.lucene.util.TestPagedBytes.testDataInputOutput2
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6:8B5FCA82AA91D006]:0)
at java.util.Arrays.copyOf(Arrays.java:3181)
at org.apache.lucene.util.PagedBytes.addBlock(PagedBytes.java:158)
at org.apache.lucene.util.PagedBytes.access$900(PagedBytes.java:36)
at org.apache.lucene.util.PagedBytes$PagedBytesDataOutput.writeBytes(PagedBytes.java:390)
at org.apache.lucene.util.TestPagedBytes.testDataInputOutput2(TestPagedBytes.java:116)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
FAILED: org.apache.lucene.util.TestPagedBytes.testDataInputOutput
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6:5ED01300913B26B5]:0)
at java.util.Arrays.copyOf(Arrays.java:3181)
at org.apache.lucene.util.PagedBytes.addBlock(PagedBytes.java:158)
at org.apache.lucene.util.PagedBytes.copy(PagedBytes.java:169)
at org.apache.lucene.util.TestPagedBytes.testDataInputOutput(TestPagedBytes.java:66)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=30820, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=30818, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=30897, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55) 4) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 5) Thread[id=30924, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 6) Thread[id=30892, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 7) Thread[id=30894, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 8) Thread[id=30893, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 9) Thread[id=30895, name=StoppableSearchThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:65) 10) Thread[id=30819, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=30820, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
2) Thread[id=30818, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=30897, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
4) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
5) Thread[id=30924, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
6) Thread[id=30892, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
7) Thread[id=30894, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
8) Thread[id=30893, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
9) Thread[id=30895, name=StoppableSearchThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:65)
10) Thread[id=30819, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([243C1376CCE7A90E]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 2) Thread[id=30892, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 3) Thread[id=30894, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 4) Thread[id=30893, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.common.util.ByteUtils.UTF16toUTF8(ByteUtils.java:90) at org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:840) at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 5) Thread[id=30895, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:897) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957) at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54) 6) Thread[id=30819, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=30923, name=StoppableCommitThread-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
2) Thread[id=30892, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
3) Thread[id=30894, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
4) Thread[id=30893, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.common.util.ByteUtils.UTF16toUTF8(ByteUtils.java:90)
at org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:840)
at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
5) Thread[id=30895, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
at org.apache.log4j.Category.callAppenders(Category.java:206)
at org.apache.log4j.Category.forcedLog(Category.java:391)
at org.apache.log4j.Category.log(Category.java:856)
at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:897)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
6) Thread[id=30819, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([243C1376CCE7A90E]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([243C1376CCE7A90E:AC682CAC621BC4F6]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:222)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 898 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexingSequenceNumbers
[junit4] 2> Jul 09, 2017 10:03:03 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1072,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
[junit4] 2> at org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4689)
[junit4] 2> at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3279)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3411)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3376)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:21 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1071,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:18 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1078,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:13 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1069,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:08 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1075,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:08 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1073,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:38 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1077,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:38 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1074,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:03:13 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1070,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] 2> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] 2> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] 2> at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:14:13 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #1735,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:703)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:683)
[junit4] 2> Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at java.util.Arrays.copyOfRange(Arrays.java:3664)
[junit4] 2> at java.lang.String.<init>(String.java:207)
[junit4] 2> at java.lang.StringBuilder.toString(StringBuilder.java:407)
[junit4] 2> at org.apache.lucene.store.BufferedChecksumIndexInput.<init>(BufferedChecksumIndexInput.java:34)
[junit4] 2> at org.apache.lucene.codecs.CodecUtil.checksumEntireFile(CodecUtil.java:520)
[junit4] 2> at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.checkIntegrity(Lucene50PostingsReader.java:1283)
[junit4] 2> at org.apache.lucene.codecs.memory.FSTOrdTermsReader.checkIntegrity(FSTOrdTermsReader.java:875)
[junit4] 2> at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.checkIntegrity(PerFieldPostingsFormat.java:348)
[junit4] 2> at org.apache.lucene.index.CodecReader.checkIntegrity(CodecReader.java:286)
[junit4] 2> at org.apache.lucene.index.MergeReaderWrapper.checkIntegrity(MergeReaderWrapper.java:178)
[junit4] 2> at org.apache.lucene.index.FilterLeafReader.checkIntegrity(FilterLeafReader.java:408)
[junit4] 2> at org.apache.lucene.index.SlowCodecReaderWrapper.wrap(SlowCodecReaderWrapper.java:54)
[junit4] 2> at org.apache.lucene.index.MockRandomMergePolicy$MockRandomOneMerge.wrapForMerge(MockRandomMergePolicy.java:179)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4375)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] 2> at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:661)
[junit4] 2>
[junit4] 2> Jul 09, 2017 10:14:13 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> WARNING: Uncaught exception in thread: Thread[Thread-1076,5,TGRP-TestIndexingSequenceNumbers]
[junit4] 2> java.lang.OutOfMemoryError: Java heap space
[junit4] 2> at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] 2> at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] 2> at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] 2> at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)
[junit4] 2>
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexingSequenceNumbers -Dtests.method=testStressConcurrentCommit -Dtests.seed=59EC404FA001EC6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=de-CH -Dtests.timezone=Etc/GMT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 1630s J0 | TestIndexingSequenceNumbers.testStressConcurrentCommit <<<
[junit4] > Throwable #1: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers.testStressConcurrentCommit(TestIndexingSequenceNumbers.java:230)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1215, name=Thread-1072, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
[junit4] > at org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4689)
[junit4] > at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3279)
[junit4] > at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3411)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3376)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1216, name=Thread-1073, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1218, name=Thread-1075, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1212, name=Thread-1069, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1213, name=Thread-1070, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1221, name=Thread-1078, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #8: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1220, name=Thread-1077, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:210)Throwable #9: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1214, name=Thread-1071, state=RUNNABLE, group=TGRP-TestIndexingSequenceNumbers]
[junit4] > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:218)
[junit4] > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:896)
[junit4] > at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:910)
[junit4] > at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3375)
[junit4] > at org.apache.lucene.index.TestIndexingSequenceNumbers$2.run(TestIndexingSequenceNumbers.java:186)
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([59EC404FA001EC6]:0)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread$IntBlockAllocator.getIntBlock(DocumentsWriterPerThread.java:624)
[junit4] > at org.apache.lucene.util.IntBlockPool.nextBuffer(IntBlockPool.java:155)
[junit4] > at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:159)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.proce
[...truncated too long message...]
a:241)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 3) Thread[id=30894, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 4) Thread[id=30893, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.ByteUtils.UTF16toUTF8(ByteUtils.java:90)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:840)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 5) Thread[id=30895, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
[junit4] > at java.io.PrintStream.flush(PrintStream.java:338)
[junit4] > at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
[junit4] > at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
[junit4] > at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
[junit4] > at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
[junit4] > at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
[junit4] > at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
[junit4] > at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
[junit4] > at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
[junit4] > at org.apache.log4j.Category.callAppenders(Category.java:206)
[junit4] > at org.apache.log4j.Category.forcedLog(Category.java:391)
[junit4] > at org.apache.log4j.Category.log(Category.java:856)
[junit4] > at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:897)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
[junit4] > at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
[junit4] > at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
[junit4] > 6) Thread[id=30819, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[243C1376CCE7A90E]-SendThread(127.0.0.1:35987), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > at __randomizedtesting.SeedInfo.seed([243C1376CCE7A90E]:0)
[junit4] Completed [502/727 (1!)] on J2 in 122.15s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 8199 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:818: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid28178.hprof
Total time: 263 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1349 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1349/
10 tests failed.
FAILED: org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings
Error Message:
startOffset must be non-negative, and endOffset must be >= startOffset, and offsets must not go backwards startOffset=15,endOffset=24,lastStartOffset=18 for field 'dummy'
Stack Trace:
java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, and offsets must not go backwards startOffset=15,endOffset=24,lastStartOffset=18 for field 'dummy'
at __randomizedtesting.SeedInfo.seed([32EF03E02BE29DAF:58B4BCF172ACBD5C]:0)
at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:769)
at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:650)
at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:540)
at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:883)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest: 1) Thread[id=3539, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=3540, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:
1) Thread[id=3539, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=3540, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=3539, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2) Thread[id=3540, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest] at java.io.PrintStream.println(PrintStream.java:805) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=3539, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
2) Thread[id=3540, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.io.PrintStream.println(PrintStream.java:805)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=48245, name=zkCallback-1581-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=48244, name=zkCallback-1581-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=48192, name=StoppableCommitThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1124) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1110) at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:646) at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49) 4) Thread[id=48217, name=zkCallback-1581-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=48215, name=StoppableCommitThread-SendThread(127.0.0.1:42281), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 6) Thread[id=48243, name=zkCallback-1581-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=48246, name=zkCallback-1581-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=48216, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=48245, name=zkCallback-1581-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=48244, name=zkCallback-1581-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=48192, name=StoppableCommitThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1124)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1110)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:646)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
4) Thread[id=48217, name=zkCallback-1581-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=48215, name=StoppableCommitThread-SendThread(127.0.0.1:42281), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
6) Thread[id=48243, name=zkCallback-1581-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=48246, name=zkCallback-1581-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=48216, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=48245, name=zkCallback-1581-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=48244, name=zkCallback-1581-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=48217, name=zkCallback-1581-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=48215, name=StoppableCommitThread-SendThread(127.0.0.1:42281), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 5) Thread[id=48243, name=zkCallback-1581-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=48246, name=zkCallback-1581-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=48245, name=zkCallback-1581-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=48244, name=zkCallback-1581-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=48217, name=zkCallback-1581-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=48215, name=StoppableCommitThread-SendThread(127.0.0.1:42281), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
5) Thread[id=48243, name=zkCallback-1581-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=48246, name=zkCallback-1581-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03:DCB481A0D3E707FB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03:DCB481A0D3E707FB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03:DCB481A0D3E707FB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest: 1) Thread[id=1363, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=1362, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:
1) Thread[id=1363, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=1362, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1363, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128) at java.io.PrintStream.write(PrintStream.java:480) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) at java.io.PrintStream.write(PrintStream.java:527) at java.io.PrintStream.print(PrintStream.java:669) at java.io.PrintStream.println(PrintStream.java:823) at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748) at java.lang.Throwable.printStackTrace(Throwable.java:658) at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121) 2) Thread[id=1362, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:615) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1363, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)
at java.io.PrintStream.write(PrintStream.java:480)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
at java.io.PrintStream.write(PrintStream.java:527)
at java.io.PrintStream.print(PrintStream.java:669)
at java.io.PrintStream.println(PrintStream.java:823)
at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
at java.lang.Throwable.printStackTrace(Throwable.java:658)
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
2) Thread[id=1362, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:615)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
Build Log:
[...truncated 2978 lines...]
[junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
[junit4] 2> TEST FAIL: useCharFilter=false text='\ua8ad\ua89e\ua8ba\ua888\ua8dd v ekpmhl \u30ee\u30b8\u30bf\u30ec\u30d6 \ud928\udf7f\u0a9c qyixp qecVut \ud800\udd7f &m\ud87a\udd04\ue73es'
[junit4] 2> Exception from random analyzer:
[junit4] 2> charfilters=
[junit4] 2> org.apache.lucene.analysis.charfilter.MappingCharFilter(org.apache.lucene.analysis.charfilter.NormalizeCharMap@5f2b7e9b, java.io.StringReader@94d4039)
[junit4] 2> tokenizer=
[junit4] 2> org.apache.lucene.analysis.standard.StandardTokenizer()
[junit4] 2> filters=
[junit4] 2> org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@3d37a71 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,termFrequency=1)
[junit4] 2> org.apache.lucene.analysis.cjk.CJKBigramFilter(ValidatingTokenFilter@64a7d776 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,termFrequency=1, 36)
[junit4] 2> org.apache.lucene.analysis.en.EnglishPossessiveFilter(ValidatingTokenFilter@1b0bd5b0 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,termFrequency=1)
[junit4] 2> org.apache.lucene.analysis.commongrams.CommonGramsQueryFilter(CommonGramsFilter@61b14de term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,termFrequency=1)
[junit4] 2> offsetsAreCorrect=false
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestRandomChains -Dtests.method=testRandomChainsWithLargeStrings -Dtests.seed=32EF03E02BE29DAF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ar-DZ -Dtests.timezone=PLT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.73s J2 | TestRandomChains.testRandomChainsWithLargeStrings <<<
[junit4] > Throwable #1: java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, and offsets must not go backwards startOffset=15,endOffset=24,lastStartOffset=18 for field 'dummy'
[junit4] > at __randomizedtesting.SeedInfo.seed([32EF03E02BE29DAF:58B4BCF172ACBD5C]:0)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:769)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:431)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:393)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:239)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:481)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1715)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
[junit4] > at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
[junit4] > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:650)
[junit4] > at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:540)
[junit4] > at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:883)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/common/test/J2/temp/lucene.analysis.core.TestRandomChains_32EF03E02BE29DAF-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {dummy=DFR GB1}, locale=ar-DZ, timezone=PLT
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=212294928,total=287309824
[junit4] 2> NOTE: All tests run in this JVM: [TestTeeSinkTokenFilter, TestApostropheFilterFactory, TestCJKBigramFilter, TestCustomAnalyzer, TestGermanStemFilter, TestCJKBigramFilterFactory, TestRussianAnalyzer, TestRandomChains]
[junit4] Completed [77/280 (1!)] on J2 in 92.48s, 2 tests, 1 error <<< FAILURES!
[...truncated 8339 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_54E0BE7A7D1B6A03-001/init-core-data-001
[junit4] 2> 32647 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=18 numCloses=18
[junit4] 2> 32648 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 32650 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 32650 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /h/b
[junit4] 2> 33394 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 34336 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 34641 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
[junit4] 2> 34674 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 34740 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 35031 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_51275_hdfs____na6yvy/webapp
[junit4] 2> 36095 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51275
[junit4] 2> 37028 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 37036 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 37064 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_53838_datanode____3on5wm/webapp
[junit4] 2> 37667 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:53838
[junit4] 2> 38433 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 38434 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 38456 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_39281_datanode____yycr1o/webapp
[junit4] 2> 38842 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[54E0BE7A7D1B6A03]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39281
[junit4] 2> 39830 INFO (IPC Server handler 7 on 40718) [ ] BlockStateChange BLOCK* processReport: from storage DS-be2158a4-1552-4baa-a4f7-689270a37b0f node DatanodeRegistration(127.0.0.1:59033, datanodeUuid=8000fa3a-7d29-4be7-bb40-9385da728a2e, infoPort=39484, infoSecurePort=0, ipcPort=33497, storageInfo=lv=-56;cid=testClusterID;nsid=750727030;c=0), blocks: 0, hasStaleStorage: true, processing time: 3 msecs
[junit4] 2> 39830 INFO (IPC Server handler 8 on 40718) [ ] BlockStateChange BLOCK* processReport: from storage DS-90634a69-0aed-4d0d-9f8c-2f61cafbe77d node DatanodeRegistration(127.0.0.1:33800, datanodeUuid=7add9fcd-ddc1-467a-9a93-fd0fda79389f, infoPort=47325, infoSecurePort=0, ipcPort=40303, storageInfo=lv=-56;cid=testClusterID;nsid=750727030;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 39831 INFO (IPC Server handler 7 on 40718) [ ] BlockStateChange BLOCK* processReport: from storage DS-450342e2-a396-4d20-bf8c-8e376d2db35c node DatanodeRegistration(127.0.0.1:59033, datanodeUuid=8000fa3a-7d29-4be7-bb40-9385da728a2e, infoPort=39484, infoSecurePort=0, ipcPort=33497, storageInfo=lv=-56;cid=testClusterID;nsid=750727030;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 39831 INFO (IPC Server handler 8 on 40718) [ ] BlockStateChange BLOCK* processReport: from storage DS-ae2ead16-021a-45de-83b5-7c997be4e296 node DatanodeRegistration(127.0.0.1:33800, datanodeUuid=7add9fcd-ddc1-467a-9a93-fd0fda79389f, infoPort=47325, infoSecurePort=0, ipcPort=40303, storageInfo=lv=-56;cid=testClusterID;nsid=750727030;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 39911 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 39912 INFO (Thread-569) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 39913 INFO (Thread-569) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 40002 ERROR (Thread-569) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 40012 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.ZkTestServer start zk server on port:36205
[junit4] 2> 40141 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 40146 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 40149 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 40152 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 40153 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 40155 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 40157 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 40158 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 40161 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 40164 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 40167 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 40186 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 40299 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 40301 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@b9ac60d{/h/b,null,AVAILABLE}
[junit4] 2> 40301 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@390203c2{SSL,[ssl, http/1.1]}{127.0.0.1:53388}
[junit4] 2> 40302 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.e.j.s.Server Started @42717ms
[junit4] 2> 40302 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40718/hdfs__localhost_40718__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_54E0BE7A7D1B6A03-001_tempDir-002_control_data, replicaType=NRT, hostContext=/h/b, hostPort=53388, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_54E0BE7A7D1B6A03-001/control-001/cores}
[junit4] 2> 40302 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 40302 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 40302 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 40302 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 40302 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-08T10:45:39.390Z
[junit4] 2> 40314 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 40314 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_54E0BE7A7D1B6A03-001/control-001/solr.xml
[junit4] 2> 40320 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@41845ee4, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 40333 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36205/solr
[junit4] 2> 40499 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 40501 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53388_h%2Fb
[junit4] 2> 40502 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.Overseer Overseer (id=98271935795560453-127.0.0.1:53388_h%2Fb-n_0000000000) starting
[junit4] 2> 40595 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53388_h%2Fb
[junit4] 2> 40602 INFO (OverseerStateUpdate-98271935795560453-127.0.0.1:53388_h%2Fb-n_0000000000) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 40728 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41845ee4
[junit4] 2> 40740 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41845ee4
[junit4] 2> 40740 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41845ee4
[junit4] 2> 40745 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_54E0BE7A7D1B6A03-001/control-001/cores
[junit4] 2> 41001 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 41004 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[54E0BE7A7D1B6A03]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36205/solr ready
[junit4] 2> 41074 INFO (qtp1935431545-797) [n:127.0.0.1:53388_h%2Fb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:53388_h%252Fb&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 41112 INFO (OverseerThreadFactory-123-thread-1-processing-n:127.0.0.1:53388_h%2Fb) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 41270 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 41271 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 41388 INFO (zkCallback-33-thread-1-processing-n:127.0.0.1:53388_h%2Fb) [n:127.0.0.1:53388_h%2Fb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 42350 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 42375 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 42482 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 42530 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 42530 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41845ee4
[junit4] 2> 42537 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40718/solr_hdfs_home
[junit4] 2> 42537 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 42537 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 42537 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_54E0BE7A7D1B6A03-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:40718/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 42541 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40718/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 42557 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 42557 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 42557 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 43521 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 43528 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40718/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 43568 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40718/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 43575 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 43575 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 43580 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 43582 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=60.53125, floorSegmentMB=1.291015625, forceMergeDeletesPctAllowed=16.225737447333184, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 43851 INFO (IPC Server handler 3 on 40718) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:59033 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-ae2ead16-021a-45de-83b5-7c997be4e296:NORMAL:127.0.0.1:33800|RBW], ReplicaUC[[DISK]DS-be2158a4-1552-4baa-a4f7-689270a37b0f:NORMAL:127.0.0.1:59033|RBW]]} size 72
[junit4] 2> 43852 INFO (IPC Server handler 2 on 40718) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33800 is added to blk_1073741825_1001 size 72
[junit4] 2> 44258 WARN (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 44338 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 44338 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 44338 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 44363 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 44363 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 44373 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 44473 INFO (qtp1935431545-794) [n:127.0.0.1:53388_h%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7668e304[control_collection_shard1_replica_n1] mai
[...truncated too long message...]
l(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=48215, name=StoppableCommitThread-SendThread(127.0.0.1:42281), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 6) Thread[id=48243, name=zkCallback-1581-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=48246, name=zkCallback-1581-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=48216, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=48245, name=zkCallback-1581-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=48244, name=zkCallback-1581-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=48217, name=zkCallback-1581-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=48215, name=StoppableCommitThread-SendThread(127.0.0.1:42281), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 5) Thread[id=48243, name=zkCallback-1581-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=48246, name=zkCallback-1581-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([54E0BE7A7D1B6A03]:0)
[junit4] Completed [725/727 (3!)] on J0 in 127.51s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 43227 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1348 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1348/
6 tests failed.
FAILED: org.apache.lucene.index.TestIndexSorting.testRandom3
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([6365789E74A119D0]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.index.TestIndexSorting
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([6365789E74A119D0]:0)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([DB34AC9D0CDE7527:53609347A22218DF]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest: 1) Thread[id=9592, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=9591, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:
1) Thread[id=9592, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=9591, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([DB34AC9D0CDE7527]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=9592, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2) Thread[id=9591, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest] at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=9592, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
2) Thread[id=9591, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
at __randomizedtesting.SeedInfo.seed([DB34AC9D0CDE7527]:0)
FAILED: org.apache.solr.cloud.TestTlogReplica.testRecovery
Error Message:
Can not find doc 8 in https://127.0.0.1:38984/solr
Stack Trace:
java.lang.AssertionError: Can not find doc 8 in https://127.0.0.1:38984/solr
at __randomizedtesting.SeedInfo.seed([DB34AC9D0CDE7527:1AC4D531218EBF80]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.TestTlogReplica.checkRTG(TestTlogReplica.java:868)
at org.apache.solr.cloud.TestTlogReplica.testRecovery(TestTlogReplica.java:589)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 1906 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexSorting
[junit4] 2> júl 07, 2017 7:20:28 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexSorting
[junit4] 2> 1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> 2) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 3) Thread[id=14587, name=TEST-TestIndexSorting.testRandom3-seed#[6365789E74A119D0], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at sun.nio.ch.NativeThread.current(Native Method)
[junit4] 2> at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46)
[junit4] 2> at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:284)
[junit4] 2> at sun.nio.ch.FileChannelImpl.position(FileChannelImpl.java:50)
[junit4] 2> at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
[junit4] 2> at org.apache.lucene.mockfile.FilterSeekableByteChannel.position(FilterSeekableByteChannel.java:74)
[junit4] 2> at org.apache.lucene.store.SimpleFSDirectory$SimpleFSIndexInput.readInternal(SimpleFSDirectory.java:171)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54)
[junit4] 2> at org.apache.lucene.store.DataInput.readInt(DataInput.java:101)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183)
[junit4] 2> at org.apache.lucene.store.DataInput.readLong(DataInput.java:157)
[junit4] 2> at org.apache.lucene.store.BufferedIndexInput.readLong(BufferedIndexInput.java:196)
[junit4] 2> at org.apache.lucene.store.MockIndexInputWrapper.readLong(MockIndexInputWrapper.java:171)
[junit4] 2> at org.apache.lucene.store.IndexInput$1.readLong(IndexInput.java:150)
[junit4] 2> at org.apache.lucene.util.packed.DirectReader$DirectPackedReader64.get(DirectReader.java:343)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$22.nextValue(Lucene70DocValuesProducer.java:1271)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextValue(DocValuesConsumer.java:446)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.writeValues(Lucene70DocValuesConsumer.java:162)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedNumericField(Lucene70DocValuesConsumer.java:530)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
[junit4] 2> at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] 2> at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
[junit4] 2> at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
[junit4] 2> at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2301)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4) Thread[id=14586, name=SUITE-TestIndexSorting-seed#[6365789E74A119D0], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=6365789E74A119D0 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sk -Dtests.timezone=America/Guayaquil -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 7199s J2 | TestIndexSorting.testRandom3 <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([6365789E74A119D0]:0)
[junit4] 2> júl 07, 2017 7:20:28 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> júl 07, 2017 7:20:48 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.index.TestIndexSorting:
[junit4] 2> 1) Thread[id=14587, name=TEST-TestIndexSorting.testRandom3-seed#[6365789E74A119D0], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> at org.apache.lucene.codecs.memory.DirectDocValuesProducer$7.setDocument(DirectDocValuesProducer.java:489)
[junit4] 2> at org.apache.lucene.index.LegacySortedNumericDocValuesWrapper.nextDoc(LegacySortedNumericDocValuesWrapper.java:53)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$SortedNumericDocValuesSub.nextDoc(DocValuesConsumer.java:359)
[junit4] 2> at org.apache.lucene.index.DocIDMerger$SortedDocIDMerger.next(DocIDMerger.java:176)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer$3$1.nextDoc(DocValuesConsumer.java:414)
[junit4] 2> at org.apache.lucene.codecs.LegacyDocValuesIterables$6$1.next(LegacyDocValuesIterables.java:295)
[junit4] 2> at org.apache.lucene.codecs.LegacyDocValuesIterables$6$1.next(LegacyDocValuesIterables.java:283)
[junit4] 2> at org.apache.lucene.codecs.memory.DirectDocValuesConsumer$1$1.next(DirectDocValuesConsumer.java:330)
[junit4] 2> at org.apache.lucene.codecs.memory.DirectDocValuesConsumer$1$1.next(DirectDocValuesConsumer.java:315)
[junit4] 2> at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.addNumericFieldValues(DirectDocValuesConsumer.java:119)
[junit4] 2> at org.apache.lucene.codecs.memory.DirectDocValuesConsumer.addSortedNumericField(DirectDocValuesConsumer.java:262)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedNumericField(DocValuesConsumer.java:371)
[junit4] 2> at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:143)
[junit4] 2> at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:151)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:181)
[junit4] 2> at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:125)
[junit4] 2> at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4390)
[junit4] 2> at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4030)
[junit4] 2> at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] 2> at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2233)
[junit4] 2> at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
[junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1718)
[junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1460)
[junit4] 2> at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2301)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> júl 07, 2017 7:20:48 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=14587, name=TEST-TestIndexSorting.testRandom3-seed#[6365789E74A119D0], state=RUNNABLE, group=TGRP-TestIndexSorting]
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexSorting_6365789E74A119D0-001
[junit4] 2> Jul 07, 2017 12:20:48 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=Lucene50(blocksize=128)}, docValues:{multi_valued_double=DocValuesFormat(name=Lucene70), multi_valued_long=DocValuesFormat(name=Direct), bytes=DocValuesFormat(name=Lucene70), double=DocValuesFormat(name=Lucene70), foo=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene70), multi_valued_int=DocValuesFormat(name=Memory), float=DocValuesFormat(name=Asserting), multi_valued_bytes=DocValuesFormat(name=Lucene70), multi_valued_float=DocValuesFormat(name=Direct), int=DocValuesFormat(name=Lucene70), long=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=266, maxMBSortInHeap=5.045152472310973, sim=RandomSimilarity(queryNorm=false): {id=DFR I(ne)1}, locale=sk, timezone=America/Guayaquil
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=1,free=164175384,total=444596224
[junit4] 2> NOTE: All tests run in this JVM: [TestIndexWriterMergePolicy, TestTermScorer, TestIntArrayDocIdSet, Test2BBinaryDocValues, TestDirectoryReader, TestMathUtil, TestMultiTermsEnum, TestStressIndexing2, TestAttributeSource, TestTermVectors, TestSegmentTermDocs, TestFieldReuse, TestDemoParallelLeafReader, TestForUtil, TestRegexpQuery, TestSimpleFSDirectory, TestVersion, TestFieldsReader, TestBoolean2ScorerSupplier, TestRamUsageEstimator, TestSortedSetSortField, TestSpanTermQuery, TestSetOnce, TestSpanNearQuery, TestMixedCodecs, TestFilterDirectory, Test2BPagedBytes, TestPrefixQuery, TestConjunctions, TestMultiFields, TestStressAdvance, TestLucene70NormsFormat, TestMultiset, TestSloppyMath, TestSleepingLockWrapper, TestIndexWriterMerging, TestBufferedIndexInput, TestIndexWriterMaxDocs, TestFieldCacheRewriteMethod, TestRollingBuffer, MultiCollectorTest, TestComplexExplanationsOfNonMatches, TestSearchForDuplicates, TestCharacterUtils, TestTimSorterWorstCase, TestPackedInts, TestIntsRef, TestNIOFSDirectory, TestSpanBoostQuery, TestIndexCommit, TestField, TestFastCompressionMode, TestShardSearching, TestBlockPostingsFormat, TestTopDocsCollector, TestNRTReaderWithThreads, TestMutablePointsReaderUtils, Test2BSortedDocValuesOrds, TestPolygon2D, TestIndexManyDocuments, TestSpans, TestIOUtils, TestEarlyTerminatingSortingCollector, TestConjunctionDISI, TestDisjunctionMaxQuery, TestLongPostings, TestPayloads, TestByteBlockPool, TestLucene50StoredFieldsFormatHighCompression, TestSortRescorer, TestSpanSearchEquivalence, TestMultiLevelSkipList, TestCachingTokenFilter, TestIndexSearcher, TestMinShouldMatch2, TestAutomaton, TestPerFieldDocValuesFormat, TestIndexWriterReader, TestFSTs, TestMultiMMap, TestBooleanOr, TestBasics, TestNRTThreads, TestDuelingCodecs, TestAtomicUpdate, TestStandardAnalyzer, TestIndexWriterCommit, TestTimeLimitingCollector, TestSearchWithThreads, TestSearchAfter, TestConcurrentMergeScheduler, TestHighCompressionMode, TestTopDocsMerge, TestStressIndexing, TestRollingUpdates, TestSnapshotDeletionPolicy, TestTieredMergePolicy, TestIndexWriterForceMerge, TestPerFieldPostingsFormat, TestArrayUtil, TestLevenshteinAutomata, TestByteSlices, TestConsistentFieldNumbers, TestSimpleExplanations, TestLockFactory, TestSegmentMerger, TestPhraseQuery, TestCollectionUtil, TestBagOfPositions, TestFixedBitSet, TestComplexExplanations, TestRegexpRandom2, TestSegmentReader, TestParallelCompositeReader, TestPagedBytes, TestTransactions, TestCrash, TestPerFieldPostingsFormat2, TestSort, TestToken, TestSimpleSearchEquivalence, TestDocValuesIndexing, TestSimilarityBase, TestHugeRamFile, TestBooleanMinShouldMatch, TestTermsEnum2, TestDeterminizeLexicon, TestIndexWriterOnDiskFull, TestCustomNorms, TestUTF32ToUTF8, TestSpanExplanations, TestLazyProxSkipping, TestDeterminism, TestMergeSchedulerExternal, TestBagOfPostings, TestTermVectorsWriter, TestCodecs, TestCharTermAttributeImpl, TestBytesRefHash, TestNearSpansOrdered, TestSameScoresWithThreads, TestTermRangeQuery, TestCrashCausesCorruptIndex, TestScorerPerf, TestLiveFieldValues, TestForTooMuchCloning, TestFuzzyQuery, TestDoc, TestWildcard, TestSimilarity2, TestExternalCodecs, TestNoDeletionPolicy, TestSimpleExplanationsOfNonMatches, TestTermVectorsReader, TestPerSegmentDeletes, TestPrefixInBooleanQuery, TestSegmentTermEnum, TestUniqueTermCount, TestUnicodeUtil, TestSortRandom, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestMaxTermFrequency, TestIndexableField, TestSentinelIntSet, TestCheckIndex, TestPayloadsOnVectors, TestCompiledAutomaton, TestSubScorerFreqs, TestMultiTermQueryRewrites, TestDateSort, TestSpanExplanationsOfNonMatches, TestSmallFloat, TestDateTools, TestSameTokenSamePosition, TestCachingCollector, TestTotalHitCountCollector, TestNoMergePolicy, TestIndexWriterLockRelease, TestTwoPhaseCommitTool, TestCodecHoldsOpenFiles, TestSimpleAttributeImpl, Test2BPositions, Test2BPostingsBytes, TestByteArrayDataInput, TestCharArrayMap, TestDelegatingAnalyzerWrapper, TestPackedTokenAttributeImpl, TestLucene50TermVectorsFormat, TestExceedMaxTermLength, TestFieldInvertState, TestFilterCodecReader, TestFilterDirectoryReader, TestIndexSorting]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.seed=6365789E74A119D0 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sk -Dtests.timezone=America/Guayaquil -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J2 | TestIndexSorting (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([6365789E74A119D0]:0)
[junit4] Completed [454/454 (1!)] on J2 in 7220.16s, 15 tests, 2 errors <<< FAILURES!
[...truncated 9946 lines...]
[junit4] Suite: org.apache.solr.cloud.TestTlogReplica
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/init-core-data-001
[junit4] 2> 630700 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 630701 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 630703 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001
[junit4] 2> 630703 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 630703 INFO (Thread-852) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 630703 INFO (Thread-852) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 630711 ERROR (Thread-852) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 630803 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:59546
[junit4] 2> 630811 INFO (jetty-launcher-494-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 630811 INFO (jetty-launcher-494-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 630814 INFO (jetty-launcher-494-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36b787f0{/solr,null,AVAILABLE}
[junit4] 2> 630814 INFO (jetty-launcher-494-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7d5e3118{/solr,null,AVAILABLE}
[junit4] 2> 630816 INFO (jetty-launcher-494-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@171ea772{SSL,[ssl, http/1.1]}{127.0.0.1:50242}
[junit4] 2> 630816 INFO (jetty-launcher-494-thread-2) [ ] o.e.j.s.Server Started @666461ms
[junit4] 2> 630816 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50242}
[junit4] 2> 630816 ERROR (jetty-launcher-494-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 630816 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 630816 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 630816 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 630817 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-07T14:28:59.325Z
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@57fd4143{SSL,[ssl, http/1.1]}{127.0.0.1:38984}
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.e.j.s.Server Started @666463ms
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38984}
[junit4] 2> 630818 ERROR (jetty-launcher-494-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 630818 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-07-07T14:28:59.326Z
[junit4] 2> 630819 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 630827 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 630830 INFO (jetty-launcher-494-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59546/solr
[junit4] 2> 630835 INFO (jetty-launcher-494-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59546/solr
[junit4] 2> 630939 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 630941 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50242_solr
[junit4] 2> 630942 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.c.Overseer Overseer (id=98267151676997636-127.0.0.1:50242_solr-n_0000000000) starting
[junit4] 2> 630946 INFO (jetty-launcher-494-thread-1) [n:127.0.0.1:38984_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 630948 INFO (jetty-launcher-494-thread-1) [n:127.0.0.1:38984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38984_solr
[junit4] 2> 630954 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50242_solr
[junit4] 2> 630956 INFO (zkCallback-502-thread-1-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 630956 INFO (zkCallback-506-thread-1-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 630961 INFO (zkCallback-506-thread-2-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 630961 INFO (OverseerStateUpdate-98267151676997636-127.0.0.1:50242_solr-n_0000000000) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 631025 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50242.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 631025 INFO (jetty-launcher-494-thread-1) [n:127.0.0.1:38984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38984.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 631030 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50242.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 631031 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50242.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 631031 INFO (jetty-launcher-494-thread-1) [n:127.0.0.1:38984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38984.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 631031 INFO (jetty-launcher-494-thread-1) [n:127.0.0.1:38984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38984.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 631032 INFO (jetty-launcher-494-thread-2) [n:127.0.0.1:50242_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001/node2/.
[junit4] 2> 631032 INFO (jetty-launcher-494-thread-1) [n:127.0.0.1:38984_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001/node1/.
[junit4] 2> 631078 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 631080 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59546/solr ready
[junit4] 2> 631083 INFO (SUITE-TestTlogReplica-seed#[DB34AC9D0CDE7527]-worker) [ ] o.a.s.c.TestTlogReplica Using legacyCloud?: true
[junit4] 2> 631111 INFO (qtp1310427979-3656) [n:127.0.0.1:38984_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=true&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 631112 INFO (qtp1310427979-3656) [n:127.0.0.1:38984_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=true&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 631131 INFO (TEST-TestTlogReplica.testRealTimeGet-seed#[DB34AC9D0CDE7527]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRealTimeGet
[junit4] 2> 631133 INFO (qtp1310427979-3657) [n:127.0.0.1:38984_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_real_time_get&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 631135 INFO (OverseerThreadFactory-1092-thread-1-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_real_time_get
[junit4] 2> 631246 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=tlog_replica_test_real_time_get_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 631247 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 631274 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=tlog_replica_test_real_time_get_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 631275 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 631384 INFO (zkCallback-506-thread-2-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 631384 INFO (zkCallback-502-thread-2-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 632277 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 632287 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_real_time_get_shard1_replica_t2] Schema name=minimal
[junit4] 2> 632290 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 632290 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_real_time_get_shard1_replica_t2' using configuration from collection tlog_replica_test_real_time_get, trusted=true
[junit4] 2> 632291 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38984.solr.core.tlog_replica_test_real_time_get.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_real_time_get.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 632291 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 632291 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_real_time_get_shard1_replica_t2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001/node1/tlog_replica_test_real_time_get_shard1_replica_t2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001/node1/./tlog_replica_test_real_time_get_shard1_replica_t2/data/]
[junit4] 2> 632292 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 632300 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_real_time_get_shard1_replica_t1] Schema name=minimal
[junit4] 2> 632303 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 632303 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_real_time_get_shard1_replica_t1' using configuration from collection tlog_replica_test_real_time_get, trusted=true
[junit4] 2> 632303 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50242.solr.core.tlog_replica_test_real_time_get.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_real_time_get.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c3051e7
[junit4] 2> 632304 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 632304 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_real_time_get_shard1_replica_t1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001/node2/tlog_replica_test_real_time_get_shard1_replica_t1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestTlogReplica_DB34AC9D0CDE7527-001/tempDir-001/node2/./tlog_replica_test_real_time_get_shard1_replica_t1/data/]
[junit4] 2> 632366 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 632366 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 632368 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 632368 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 632371 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 632371 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 632372 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b3dc234[tlog_replica_test_real_time_get_shard1_replica_t1] main]
[junit4] 2> 632373 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 632373 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 632373 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 632374 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 632374 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 632375 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4da0e4ed[tlog_replica_test_real_time_get_shard1_replica_t2] main]
[junit4] 2> 632376 INFO (searcherExecutor-1098-thread-1-processing-n:127.0.0.1:50242_solr x:tlog_replica_test_real_time_get_shard1_replica_t1 s:shard1 c:tlog_replica_test_real_time_get) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_real_time_get_shard1_replica_t1] Registered new searcher Searcher@3b3dc234[tlog_replica_test_real_time_get_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 632376 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572274428585181184
[junit4] 2> 632378 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 632378 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 632386 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 632387 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 632388 INFO (searcherExecutor-1097-thread-1-processing-n:127.0.0.1:38984_solr x:tlog_replica_test_real_time_get_shard1_replica_t2 s:shard1 c:tlog_replica_test_real_time_get) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_real_time_get_shard1_replica_t2] Registered new searcher Searcher@4da0e4ed[tlog_replica_test_real_time_get_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 632388 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1572274428597764096
[junit4] 2> 632487 INFO (zkCallback-506-thread-2-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 632487 INFO (zkCallback-502-thread-2-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 632887 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 632887 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 632887 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 632887 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_real_time_get_shard1_replica_t1 url=https://127.0.0.1:50242/solr START replicas=[https://127.0.0.1:38984/solr/tlog_replica_test_real_time_get_shard1_replica_t2/] nUpdates=100
[junit4] 2> 632888 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_real_time_get_shard1_replica_t1 url=https://127.0.0.1:50242/solr DONE. We have no versions. sync failed.
[junit4] 2> 632891 INFO (qtp1310427979-3650) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 632892 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 632892 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 632892 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 632892 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_real_time_get_shard1_replica_t1 stopping background replication from leader
[junit4] 2> 632897 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/ shard1
[junit4] 2> 632999 INFO (zkCallback-506-thread-2-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 632999 INFO (zkCallback-502-thread-2-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 633048 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 633051 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=tlog_replica_test_real_time_get_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1776
[junit4] 2> 633155 INFO (zkCallback-502-thread-2-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 633155 INFO (zkCallback-506-thread-2-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 633396 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.ZkController tlog_replica_test_real_time_get_shard1_replica_t2 starting background replication from leader
[junit4] 2> 633396 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 633400 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 633400 INFO (indexFetcher-1108-thread-1) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 633403 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=tlog_replica_test_real_time_get_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2156
[junit4] 2> 633405 INFO (qtp1451090140-3648) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 633405 INFO (indexFetcher-1108-thread-1) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 633405 INFO (indexFetcher-1108-thread-1) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 633405 INFO (indexFetcher-1108-thread-1) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 633405 INFO (indexFetcher-1108-thread-1) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 633407 INFO (qtp1310427979-3657) [n:127.0.0.1:38984_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 633507 INFO (zkCallback-502-thread-2-processing-n:127.0.0.1:50242_solr) [n:127.0.0.1:50242_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 633507 INFO (zkCallback-506-thread-2-processing-n:127.0.0.1:38984_solr) [n:127.0.0.1:38984_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_real_time_get/state.json] for collection [tlog_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 634408 INFO (qtp1310427979-3657) [n:127.0.0.1:38984_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_real_time_get&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=3274
[junit4] 2> 634424 INFO (qtp1310427979-3654) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2}{add=[0 (1572274430728470528)]} 0 0
[junit4] 2> 634425 INFO (qtp1451090140-3649) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=https://127.0.0.1:38984/solr/tlog_replica_test_real_time_get_shard1_replica_t2/&wt=javabin&version=2}{add=[0 (1572274430728470528)]} 0 5
[junit4] 2> 634425 INFO (qtp1310427979-3643) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 6
[junit4] 2> 634426 INFO (qtp1310427979-3656) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 634430 INFO (qtp1451090140-3648) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&_stateVer_=tlog_replica_test_real_time_get:5&omitHeader=true&shards.purpose=1&NOW=1499437742934&ids=0&isShard=true&shard.url=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 634431 INFO (qtp1310427979-3656) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/get params={qt=/get&_stateVer_=tlog_replica_test_real_time_get:5&ids=0&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 634432 INFO (qtp1310427979-3650) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 634432 INFO (qtp1451090140-3649) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1499437742940&ids=0&isShard=true&shard.url=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 634433 INFO (qtp1310427979-3650) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/get params={qt=/get&ids=0&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 634435 INFO (qtp1451090140-3644) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 634437 INFO (qtp1451090140-3655) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1499437742943&ids=0&isShard=true&shard.url=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 634437 INFO (qtp1451090140-3644) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={qt=/get&ids=0&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 634440 INFO (qtp1310427979-3645) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2}{add=[1 (1572274430747344896)]} 0 0
[junit4] 2> 634440 INFO (qtp1451090140-3642) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1572274430747344896)]} 0 1
[junit4] 2> 634441 INFO (qtp1451090140-3649) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 634441 INFO (qtp1451090140-3648) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&_stateVer_=tlog_replica_test_real_time_get:5&omitHeader=true&shards.purpose=1&NOW=1499437742948&ids=1&isShard=true&shard.url=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 634442 INFO (qtp1451090140-3649) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={qt=/get&_stateVer_=tlog_replica_test_real_time_get:5&ids=1&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 634442 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 634443 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1499437742950&ids=1&isShard=true&shard.url=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 634443 INFO (qtp1310427979-3652) [n:127.0.0.1:38984_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node1 x:tlog_replica_test_real_time_get_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t2] webapp=/solr path=/get params={qt=/get&ids=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 634444 INFO (qtp1451090140-3653) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=https://127.0.0.1:50242/solr/tlog_replica_test_real_time_get_shard1_replica_t1/
[junit4] 2> 634444 INFO (qtp1451090140-3651) [n:127.0.0.1:50242_solr c:tlog_replica_test_real_time_get s:shard1 r:core_node2 x:tlog_replica_test_real_time_get_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_real_time_get_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHe
[...truncated too long message...]
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), id=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1723, maxMBSortInHeap=6.1244590694894985, sim=RandomSimilarity(queryNorm=true): {}, locale=vi, timezone=America/Nipigon
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=4,threads=4,free=149312336,total=517472256
[junit4] 2> NOTE: All tests run in this JVM: [DocumentBuilderTest, TestUtils, SortByFunctionTest, TestManagedResource, DataDrivenBlockJoinTest, TestCloudJSONFacetJoinDomain, TestDownShardTolerantSearch, TestBlobHandler, SpellPossibilityIteratorTest, TestCloudManagedSchema, TestTrie, HardAutoCommitTest, SimpleFacetsTest, ImplicitSnitchTest, WrapperMergePolicyFactoryTest, TestSolrConfigHandlerConcurrent, StatsReloadRaceTest, TestExpandComponent, MetricUtilsTest, LeaderInitiatedRecoveryOnShardRestartTest, PrimitiveFieldTypeTest, BlockDirectoryTest, SignatureUpdateProcessorFactoryTest, CircularListTest, TestDFRSimilarityFactory, BJQParserTest, AtomicUpdatesTest, VMParamsZkACLAndCredentialsProvidersTest, SolrGraphiteReporterTest, UnloadDistributedZkTest, TestCryptoKeys, OverseerTest, TermsComponentTest, TestReplicationHandlerBackup, BasicDistributedZk2Test, TestSubQueryTransformer, TestSuggestSpellingConverter, CopyFieldTest, TestZkChroot, SolrInfoBeanTest, SolrCLIZkUtilsTest, TestFieldCacheReopen, DistributedFacetPivotLongTailTest, RequiredFieldsTest, ExitableDirectoryReaderTest, TestGraphTermsQParserPlugin, TestMinMaxOnMultiValuedField, SuggestComponentContextFilterQueryTest, TestManagedStopFilterFactory, ConfigureRecoveryStrategyTest, HdfsUnloadDistributedZkTest, TestLFUCache, PKIAuthenticationIntegrationTest, EnumFieldTest, TestPseudoReturnFields, DistributedFacetPivotWhiteBoxTest, TestCloudRecovery, TestSolrCLIRunExample, XsltUpdateRequestHandlerTest, CollectionsAPISolrJTest, BadCopyFieldTest, TestJmxIntegration, TestMaxTokenLenTokenizer, TestSolrCloudWithSecureImpersonation, NumericFieldsTest, TestTolerantUpdateProcessorRandomCloud, TestMultiValuedNumericRangeQuery, TestRuleBasedAuthorizationPlugin, PingRequestHandlerTest, V2StandaloneTest, TestStressUserVersions, TestSolrCoreProperties, MoveReplicaHDFSTest, RecoveryZkTest, TestMacroExpander, DateFieldTest, MultiTermTest, HdfsChaosMonkeySafeLeaderTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsChaosMonkeySafeLeaderTest -Dtests.seed=DB34AC9D0CDE7527 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=vi -Dtests.timezone=America/Nipigon -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J2 | HdfsChaosMonkeySafeLeaderTest (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:
[junit4] > 1) Thread[id=9592, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
[junit4] > 2) Thread[id=9591, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
[junit4] > at __randomizedtesting.SeedInfo.seed([DB34AC9D0CDE7527]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=9592, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:612)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 2) Thread[id=9591, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:381)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:241)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > at __randomizedtesting.SeedInfo.seed([DB34AC9D0CDE7527]:0)
[junit4] Completed [207/727 (2!)] on J2 in 102.52s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 44832 lines...]