You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by th...@apache.org on 2020/09/10 14:42:41 UTC
[lucene-solr] branch reference_impl_dev updated: Use a
CountDownLatch for doParallelUpdates
This is an automated email from the ASF dual-hosted git repository.
thelabdude pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl_dev by this push:
new 5cbe26a Use a CountDownLatch for doParallelUpdates
5cbe26a is described below
commit 5cbe26ae524755044da017ac268b65fe8a4e386f
Author: Timothy Potter <th...@gmail.com>
AuthorDate: Thu Sep 10 08:42:16 2020 -0600
Use a CountDownLatch for doParallelUpdates
---
.../client/solrj/impl/CloudHttp2SolrClient.java | 22 ++++++++++++++---
.../impl/CloudHttp2SolrClientWireMockTest.java | 28 ++++++++++++----------
2 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
index f4d6b99..b1ae052 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.ParWork;
@@ -120,6 +121,7 @@ public class CloudHttp2SolrClient extends BaseCloudSolrClient {
NamedList<Throwable> exceptions, NamedList<NamedList> shardResponses) {
Map<String,Throwable> tsExceptions = new ConcurrentHashMap<>();
Map<String,NamedList> tsResponses = new ConcurrentHashMap<>();
+ final CountDownLatch latch = new CountDownLatch(routes.size());
for (final Map.Entry<String, ? extends LBSolrClient.Req> entry : routes.entrySet()) {
final String url = entry.getKey();
final LBSolrClient.Req lbRequest = entry.getValue();
@@ -127,11 +129,13 @@ public class CloudHttp2SolrClient extends BaseCloudSolrClient {
try {
MDC.put("CloudSolrClient.url", url);
try {
- myClient.request(lbRequest.request, null, new UpdateOnComplete(tsResponses, url, tsExceptions));
+ myClient.request(lbRequest.request, null, new UpdateOnComplete(latch, tsResponses, url, tsExceptions));
} catch (IOException e) {
tsExceptions.put(url, e);
+ latch.countDown();
} catch (SolrServerException e) {
tsExceptions.put(url, e);
+ latch.countDown();
}
} finally {
@@ -140,7 +144,15 @@ public class CloudHttp2SolrClient extends BaseCloudSolrClient {
}
// wait until the async requests we fired off above are done
- myClient.waitForOutstandingRequests();
+ // nocommit ~ TJP: can't get this approach to work but CDL works fine,
+ // see CloudHttp2SolrClientWireMockTest#testConcurrentParallelUpdates
+ //myClient.waitForOutstandingRequests();
+ try {
+ latch.await(); // eventually the requests will timeout after the socket read timeout is reached.
+ } catch (InterruptedException e) {
+ ParWork.propegateInterrupt(e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
exceptions.addAll(tsExceptions);
shardResponses.addAll(tsResponses);
@@ -306,11 +318,13 @@ public class CloudHttp2SolrClient extends BaseCloudSolrClient {
private static class UpdateOnComplete implements Http2SolrClient.OnComplete {
+ private final CountDownLatch latch;
private final Map<String,NamedList> tsResponses;
private final String url;
private final Map<String,Throwable> tsExceptions;
- public UpdateOnComplete(Map<String,NamedList> tsResponses, String url, Map<String,Throwable> tsExceptions) {
+ public UpdateOnComplete(CountDownLatch latch, Map<String,NamedList> tsResponses, String url, Map<String,Throwable> tsExceptions) {
+ this.latch = latch;
this.tsResponses = tsResponses;
this.url = url;
this.tsExceptions = tsExceptions;
@@ -318,11 +332,13 @@ public class CloudHttp2SolrClient extends BaseCloudSolrClient {
@Override
public void onSuccess(NamedList result) {
+ latch.countDown();
tsResponses.put(url, result);
}
@Override
public void onFailure(Throwable t) {
+ latch.countDown();
tsExceptions.put(url, t);
}
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java
index 8493a3d..145d9e7 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientWireMockTest.java
@@ -28,7 +28,6 @@ import com.github.tomakehurst.wiremock.stubbing.ServeEvent;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.util.NamedList;
-import org.junit.Ignore;
import org.junit.Test;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
@@ -90,42 +89,47 @@ public class CloudHttp2SolrClientWireMockTest extends BaseSolrClientWireMockTest
@SuppressWarnings({"rawtypes", "unchecked"})
@Test
- @Ignore // nocommit ~ TJP WIP fails b/c some responses don't have both route responses
public void testConcurrentParallelUpdates() throws Exception {
// expect update requests go to both shards
stubFor(post(urlPathEqualTo(SHARD1_PATH+"/update"))
.willReturn(ok()
- .withLogNormalRandomDelay(140, 0.1)
+ .withLogNormalRandomDelay(100, 0.1)
.withHeader("Content-Type", RESPONSE_CONTENT_TYPE)
.withBody(updateRequestOk())));
stubFor(post(urlPathEqualTo(SHARD2_PATH+"/update"))
.willReturn(ok()
- .withLogNormalRandomDelay(70, 0.1)
+ .withLogNormalRandomDelay(50, 0.1)
.withHeader("Content-Type", RESPONSE_CONTENT_TYPE)
.withBody(updateRequestOk())));
List<Future<UpdateResponse>> list = new ArrayList<>();
- ExecutorService executorService = Executors.newFixedThreadPool(4);
- for (int t=0; t < 10; t++) {
+ List<Throwable> fails = new ArrayList<>();
+ ExecutorService executorService = Executors.newFixedThreadPool(5);
+ final int numRequests = 10;
+ for (int t=0; t < numRequests; t++) {
Future<UpdateResponse> responseFuture = executorService.submit(() -> {
UpdateRequest req = buildUpdateRequest(20);
- UpdateResponse resp;
+ UpdateResponse resp = null;
try {
resp = req.process(testClient, BUILT_IN_MOCK_COLLECTION);
- } catch (Exception e) {
- throw new RuntimeException(e);
+ } catch (Throwable e) {
+ fails.add(e);
}
return resp;
});
list.add(responseFuture);
}
executorService.shutdown();
- executorService.awaitTermination(3, TimeUnit.SECONDS);
+ executorService.awaitTermination(3, TimeUnit.SECONDS); // 3 secs should be ample time for a mock server
+
+ if (!fails.isEmpty()) {
+ fail("Not all requests succeeded, fails: "+fails);
+ }
List<ServeEvent> events = mockSolr.getAllServeEvents();
- // code should have sent 10 requests to each shard leader
- assertEquals(20, events.size());
+ // code should have sent numRequests requests to each shard leader
+ assertEquals(2 * numRequests, events.size());
// verify every response has 2 route responses!
for (int i=0; i < list.size(); i++) {