You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by th...@apache.org on 2020/09/10 14:42:41 UTC

[lucene-solr] branch reference_impl_dev updated: Use a CountDownLatch for doParallelUpdates

This is an automated email from the ASF dual-hosted git repository.

thelabdude pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/reference_impl_dev by this push:
     new 5cbe26a  Use a CountDownLatch for doParallelUpdates
5cbe26a is described below

commit 5cbe26ae524755044da017ac268b65fe8a4e386f
Author: Timothy Potter <th...@gmail.com>
AuthorDate: Thu Sep 10 08:42:16 2020 -0600

    Use a CountDownLatch for doParallelUpdates
---
 .../client/solrj/impl/CloudHttp2SolrClient.java    | 22 ++++++++++++++---
 .../impl/CloudHttp2SolrClientWireMockTest.java     | 28 ++++++++++++----------
 2 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
index f4d6b99..b1ae052 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.common.ParWork;
@@ -120,6 +121,7 @@ public class CloudHttp2SolrClient  extends BaseCloudSolrClient {
       NamedList<Throwable> exceptions, NamedList<NamedList> shardResponses) {
     Map<String,Throwable> tsExceptions = new ConcurrentHashMap<>();
     Map<String,NamedList> tsResponses = new ConcurrentHashMap<>();
+    final CountDownLatch latch = new CountDownLatch(routes.size());
     for (final Map.Entry<String, ? extends LBSolrClient.Req> entry : routes.entrySet()) {
       final String url = entry.getKey();
       final LBSolrClient.Req lbRequest = entry.getValue();
@@ -127,11 +129,13 @@ public class CloudHttp2SolrClient  extends BaseCloudSolrClient {
       try {
         MDC.put("CloudSolrClient.url", url);
         try {
-          myClient.request(lbRequest.request, null, new UpdateOnComplete(tsResponses, url, tsExceptions));
+          myClient.request(lbRequest.request, null, new UpdateOnComplete(latch, tsResponses, url, tsExceptions));
         } catch (IOException e) {
           tsExceptions.put(url, e);
+          latch.countDown();
         } catch (SolrServerException e) {
           tsExceptions.put(url, e);
+          latch.countDown();
         }
 
       } finally {
@@ -140,7 +144,15 @@ public class CloudHttp2SolrClient  extends BaseCloudSolrClient {
     }
 
     // wait until the async requests we fired off above are done
-    myClient.waitForOutstandingRequests();
+    // nocommit ~ TJP: can't get this approach to work but CDL works fine,
+    // see CloudHttp2SolrClientWireMockTest#testConcurrentParallelUpdates
+    //myClient.waitForOutstandingRequests();
+    try {
+      latch.await(); // eventually the requests will timeout after the socket read timeout is reached.
+    } catch (InterruptedException e) {
+      ParWork.propegateInterrupt(e);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+    }
 
     exceptions.addAll(tsExceptions);
     shardResponses.addAll(tsResponses);
@@ -306,11 +318,13 @@ public class CloudHttp2SolrClient  extends BaseCloudSolrClient {
 
   private static class UpdateOnComplete implements Http2SolrClient.OnComplete {
 
+    private final CountDownLatch latch;
     private final Map<String,NamedList> tsResponses;
     private final String url;
     private final Map<String,Throwable> tsExceptions;
 
-    public UpdateOnComplete(Map<String,NamedList> tsResponses, String url, Map<String,Throwable> tsExceptions) {
+    public UpdateOnComplete(CountDownLatch latch, Map<String,NamedList> tsResponses, String url, Map<String,Throwable> tsExceptions) {
+      this.latch = latch;
       this.tsResponses = tsResponses;
       this.url = url;
       this.tsExceptions = tsExceptions;
@@ -318,11 +332,13 @@ public class CloudHttp2SolrClient  extends BaseCloudSolrClient {
 
     @Override
     public void onSuccess(NamedList result) {
+      latch.countDown();
       tsResponses.put(url, result);
     }
 
     @Override
     public void onFailure(Throwable t) {
+      latch.countDown();
       tsExceptions.put(url, t);
     }
   }
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java
index 8493a3d..145d9e7 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java
@@ -28,7 +28,6 @@ import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.common.util.NamedList;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
@@ -90,42 +89,47 @@ public class CloudHttp2SolrClientWireMockTest extends BaseSolrClientWireMockTest
 
   @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
-  @Ignore // nocommit ~ TJP WIP fails b/c some responses don't have both route responses
   public void testConcurrentParallelUpdates() throws Exception {
     // expect update requests go to both shards
     stubFor(post(urlPathEqualTo(SHARD1_PATH+"/update"))
         .willReturn(ok()
-            .withLogNormalRandomDelay(140, 0.1)
+            .withLogNormalRandomDelay(100, 0.1)
             .withHeader("Content-Type", RESPONSE_CONTENT_TYPE)
             .withBody(updateRequestOk())));
 
     stubFor(post(urlPathEqualTo(SHARD2_PATH+"/update"))
         .willReturn(ok()
-            .withLogNormalRandomDelay(70, 0.1)
+            .withLogNormalRandomDelay(50, 0.1)
             .withHeader("Content-Type", RESPONSE_CONTENT_TYPE)
             .withBody(updateRequestOk())));
 
     List<Future<UpdateResponse>> list = new ArrayList<>();
-    ExecutorService executorService = Executors.newFixedThreadPool(4);
-    for (int t=0; t < 10; t++) {
+    List<Throwable> fails = new ArrayList<>();
+    ExecutorService executorService = Executors.newFixedThreadPool(5);
+    final int numRequests = 10;
+    for (int t=0; t < numRequests; t++) {
       Future<UpdateResponse> responseFuture = executorService.submit(() -> {
         UpdateRequest req = buildUpdateRequest(20);
-        UpdateResponse resp;
+        UpdateResponse resp = null;
         try {
           resp = req.process(testClient, BUILT_IN_MOCK_COLLECTION);
-        } catch (Exception e) {
-          throw new RuntimeException(e);
+        } catch (Throwable e) {
+          fails.add(e);
         }
         return resp;
       });
       list.add(responseFuture);
     }
     executorService.shutdown();
-    executorService.awaitTermination(3, TimeUnit.SECONDS);
+    executorService.awaitTermination(3, TimeUnit.SECONDS); // 3 secs should be ample time for a mock server
+
+    if (!fails.isEmpty()) {
+      fail("Not all requests succeeded, fails: "+fails);
+    }
 
     List<ServeEvent> events = mockSolr.getAllServeEvents();
-    // code should have sent 10 requests to each shard leader
-    assertEquals(20, events.size());
+    // code should have sent numRequests requests to each shard leader
+    assertEquals(2 * numRequests, events.size());
 
     // verify every response has 2 route responses!
     for (int i=0; i < list.size(); i++) {