You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Joe Obernberger <jo...@gmail.com> on 2020/04/28 15:50:30 UTC

Delete on 8.5.1

Hi all - I'm running this query on solr cloud 8.5.1 with the index on HDFS:

curl http://enceladus:9100/solr/PROCESSOR_LOGS/update?commit=true -H 
"Connect-Type: text/xml" --data-binary 
'<delete><query>StartTime:[2020-01-01T01:02:43Z TO 
2020-04-25T00:00:00Z]</query></delete>'

getting this response:

<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
   <int name="rf">1</int>
   <int name="status">500</int>
   <int name="QTime">54091</int>
</lst>
<lst name="error">
   <lst name="metadata">
     <str 
name="error-class">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException</str>
     <str 
name="root-error-class">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException</str>
   </lst>
   <str name="msg">2 Async exceptions during distributed update:
Error from server at 
http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/: null



request: http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/
Remote error message: Task queue processing has stalled for 20193 ms 
with 0 remaining elements to process.
Error from server at 
http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/: null



request: http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/
Remote error message: Task queue processing has stalled for 20021 ms 
with 0 remaining elements to process.</str>
   <str 
name="trace">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException: 
2 Async exceptions during distributed update:
Error from server at 
http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/: null



request: http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/
Remote error message: Task queue processing has stalled for 20193 ms 
with 0 remaining elements to process.
Error from server at 
http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/: null



request: http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/
Remote error message: Task queue processing has stalled for 20021 ms 
with 0 remaining elements to process.
         at 
org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDistribFinish(DistributedZkUpdateProcessor.java:1189)
         at 
org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1096)
         at 
org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
         at 
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
         at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:211)
         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2596)
         at 
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:802)
         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:579)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:420)
         at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:352)
         at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596)
         at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
         at 
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
         at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
         at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
         at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
         at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
         at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212)
         at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
         at 
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221)
         at 
org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)
         at 
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
         at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
         at 
org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
         at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
         at org.eclipse.jetty.server.Server.handle(Server.java:500)
         at 
org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
         at 
org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
         at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
         at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
         at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
         at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
         at 
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
         at 
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
         at java.lang.Thread.run(Thread.java:748)
</str>
   <int name="code">500</int>
</lst>
</response>

Is there a way to process this asynchronously?

-Joe


Re: Delete on 8.5.1

Posted by Joe Obernberger <jo...@gmail.com>.
Hi All - while I'm still getting the error, it does appear to work 
(still gives the error - but a search of the data then shows less 
results - so the delete is working).  In some cases, it may be necessary 
to run the query several times.

-Joe

On 4/29/2020 9:03 AM, Joe Obernberger wrote:
> Hi - I also tried deleting from solrj (8.5.1) using 
> CloudSolrClient.deleteByQuery.
>
> This results in:
>
> Error: 
> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: 
> Error from server at http://paradigm8:9100/solr/PROCESSOR_LOGS: Async 
> exception during distributed update: Error from server at 
> http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/: null
>
>
>
> request: http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/
> Remote error message: Task queue processing has stalled for 20203 ms 
> with 0 remaining elements to process.
> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: 
> Error from server at http://paradigm8:9100/solr/PROCESSOR_LOGS: Async 
> exception during distributed update: Error from server at 
> http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/: null
>
>
>
> request: http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/
> Remote error message: Task queue processing has stalled for 20203 ms 
> with 0 remaining elements to process.
>         at 
> org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
>         at 
> org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
>         at 
> org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
>         at 
> org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
>         at 
> org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
>         at 
> org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
>         at 
> org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
>         at 
> org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
>         at 
> org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
>         at 
> org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:940)
>         at 
> org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:903)
>         at 
> com.ngc.bigdata.solrsearcher.SearcherThread.doSearch(SearcherThread.java:401)
>         at 
> com.ngc.bigdata.solrsearcher.SearcherThread.run(SearcherThread.java:125)
>         at 
> com.ngc.bigdata.solrsearcher.Worker.doSearchTest(Worker.java:145)
>         at 
> com.ngc.bigdata.solrsearcher.SolrSearcher.main(SolrSearcher.java:60)
>
>
> On 4/28/2020 11:50 AM, Joe Obernberger wrote:
>> Hi all - I'm running this query on solr cloud 8.5.1 with the index on 
>> HDFS:
>>
>> curl http://enceladus:9100/solr/PROCESSOR_LOGS/update?commit=true -H 
>> "Connect-Type: text/xml" --data-binary 
>> '<delete><query>StartTime:[2020-01-01T01:02:43Z TO 
>> 2020-04-25T00:00:00Z]</query></delete>'
>>
>> getting this response:
>>
>> <?xml version="1.0" encoding="UTF-8"?>
>> <response>
>>
>> <lst name="responseHeader">
>>   <int name="rf">1</int>
>>   <int name="status">500</int>
>>   <int name="QTime">54091</int>
>> </lst>
>> <lst name="error">
>>   <lst name="metadata">
>>     <str 
>> name="error-class">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException</str>
>>     <str 
>> name="root-error-class">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException</str>
>>   </lst>
>>   <str name="msg">2 Async exceptions during distributed update:
>> Error from server at 
>> http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/: null
>>
>>
>>
>> request: http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/
>> Remote error message: Task queue processing has stalled for 20193 ms 
>> with 0 remaining elements to process.
>> Error from server at 
>> http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/: null
>>
>>
>>
>> request: http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/
>> Remote error message: Task queue processing has stalled for 20021 ms 
>> with 0 remaining elements to process.</str>
>>   <str 
>> name="trace">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException: 
>> 2 Async exceptions during distributed update:
>> Error from server at 
>> http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/: null
>>
>>
>>
>> request: http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/
>> Remote error message: Task queue processing has stalled for 20193 ms 
>> with 0 remaining elements to process.
>> Error from server at 
>> http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/: null
>>
>>
>>
>> request: http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/
>> Remote error message: Task queue processing has stalled for 20021 ms 
>> with 0 remaining elements to process.
>>         at 
>> org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDistribFinish(DistributedZkUpdateProcessor.java:1189)
>>         at 
>> org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1096)
>>         at 
>> org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>>         at 
>> org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
>>         at 
>> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:211)
>>         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2596)
>>         at 
>> org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:802)
>>         at 
>> org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:579)
>>         at 
>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:420)
>>         at 
>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:352)
>>         at 
>> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596)
>>         at 
>> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
>>         at 
>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>>         at 
>> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
>>         at 
>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
>>         at 
>> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
>>         at 
>> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607)
>>         at 
>> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
>>         at 
>> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297)
>>         at 
>> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
>>         at 
>> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
>>         at 
>> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577)
>>         at 
>> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
>>         at 
>> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212)
>>         at 
>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
>>         at 
>> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221)
>>         at 
>> org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)
>>         at 
>> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
>>         at 
>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
>>         at 
>> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
>>         at 
>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
>>         at org.eclipse.jetty.server.Server.handle(Server.java:500)
>>         at 
>> org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
>>         at 
>> org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
>>         at 
>> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
>>         at 
>> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
>>         at 
>> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
>>         at 
>> org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
>>         at 
>> org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
>>         at 
>> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
>>         at 
>> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
>>         at 
>> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
>>         at 
>> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
>>         at 
>> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388)
>>         at 
>> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
>>         at 
>> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
>>         at java.lang.Thread.run(Thread.java:748)
>> </str>
>>   <int name="code">500</int>
>> </lst>
>> </response>
>>
>> Is there a way to process this asynchronously?
>>
>> -Joe
>>

Re: Delete on 8.5.1

Posted by Joe Obernberger <jo...@gmail.com>.
Hi - I also tried deleting from solrj (8.5.1) using 
CloudSolrClient.deleteByQuery.

This results in:

Error: 
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: 
Error from server at http://paradigm8:9100/solr/PROCESSOR_LOGS: Async 
exception during distributed update: Error from server at 
http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/: null



request: http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/
Remote error message: Task queue processing has stalled for 20203 ms 
with 0 remaining elements to process.
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: 
Error from server at http://paradigm8:9100/solr/PROCESSOR_LOGS: Async 
exception during distributed update: Error from server at 
http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/: null



request: http://paradigm7:9100/solr/PROCESSOR_LOGS_shard6_replica_n20/
Remote error message: Task queue processing has stalled for 20203 ms 
with 0 remaining elements to process.
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
         at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
         at 
org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
         at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
         at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
         at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
         at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
         at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
         at 
org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:940)
         at 
org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:903)
         at 
com.ngc.bigdata.solrsearcher.SearcherThread.doSearch(SearcherThread.java:401)
         at 
com.ngc.bigdata.solrsearcher.SearcherThread.run(SearcherThread.java:125)
         at 
com.ngc.bigdata.solrsearcher.Worker.doSearchTest(Worker.java:145)
         at 
com.ngc.bigdata.solrsearcher.SolrSearcher.main(SolrSearcher.java:60)


On 4/28/2020 11:50 AM, Joe Obernberger wrote:
> Hi all - I'm running this query on solr cloud 8.5.1 with the index on 
> HDFS:
>
> curl http://enceladus:9100/solr/PROCESSOR_LOGS/update?commit=true -H 
> "Connect-Type: text/xml" --data-binary 
> '<delete><query>StartTime:[2020-01-01T01:02:43Z TO 
> 2020-04-25T00:00:00Z]</query></delete>'
>
> getting this response:
>
> <?xml version="1.0" encoding="UTF-8"?>
> <response>
>
> <lst name="responseHeader">
>   <int name="rf">1</int>
>   <int name="status">500</int>
>   <int name="QTime">54091</int>
> </lst>
> <lst name="error">
>   <lst name="metadata">
>     <str 
> name="error-class">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException</str>
>     <str 
> name="root-error-class">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException</str>
>   </lst>
>   <str name="msg">2 Async exceptions during distributed update:
> Error from server at 
> http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/: null
>
>
>
> request: http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/
> Remote error message: Task queue processing has stalled for 20193 ms 
> with 0 remaining elements to process.
> Error from server at 
> http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/: null
>
>
>
> request: http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/
> Remote error message: Task queue processing has stalled for 20021 ms 
> with 0 remaining elements to process.</str>
>   <str 
> name="trace">org.apache.solr.update.processor.DistributedUpdateProcessor$DistributedUpdatesAsyncException: 
> 2 Async exceptions during distributed update:
> Error from server at 
> http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/: null
>
>
>
> request: http://paradigm8:9100/solr/PROCESSOR_LOGS_shard2_replica_n4/
> Remote error message: Task queue processing has stalled for 20193 ms 
> with 0 remaining elements to process.
> Error from server at 
> http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/: null
>
>
>
> request: http://belinda:9100/solr/PROCESSOR_LOGS_shard10_replica_n38/
> Remote error message: Task queue processing has stalled for 20021 ms 
> with 0 remaining elements to process.
>         at 
> org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDistribFinish(DistributedZkUpdateProcessor.java:1189)
>         at 
> org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1096)
>         at 
> org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.update.processor.UpdateRequestProcessor.finish(UpdateRequestProcessor.java:80)
>         at 
> org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
>         at 
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:211)
>         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2596)
>         at 
> org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:802)
>         at 
> org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:579)
>         at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:420)
>         at 
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:352)
>         at 
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596)
>         at 
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
>         at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
>         at 
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
>         at 
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
>         at 
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
>         at 
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607)
>         at 
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
>         at 
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297)
>         at 
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
>         at 
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
>         at 
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577)
>         at 
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
>         at 
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212)
>         at 
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
>         at 
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221)
>         at 
> org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)
>         at 
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
>         at 
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
>         at 
> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
>         at 
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
>         at org.eclipse.jetty.server.Server.handle(Server.java:500)
>         at 
> org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
>         at 
> org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
>         at 
> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
>         at 
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
>         at 
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
>         at 
> org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
>         at 
> org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
>         at 
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
>         at 
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
>         at 
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
>         at 
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
>         at 
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388)
>         at 
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
>         at 
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
>         at java.lang.Thread.run(Thread.java:748)
> </str>
>   <int name="code">500</int>
> </lst>
> </response>
>
> Is there a way to process this asynchronously?
>
> -Joe
>