You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/09/04 11:16:06 UTC
[lucene-solr] branch reference_impl updated (f9b7400 -> 5ced9d4)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from f9b7400 @719 Lookup build methods can raise with file work and cause lookup build fails.
new 7f63eb8 @720 UpdateLog executor usage fix/improve.
new 7ffb68c @721 Convert some more executor usage to shared execs.
new c22e43e @722 Try and prevent UpdateLog leak.
new 072951f @723 Remove semi hack, should not be needed now that we don't miss registering a queued listener.
new 4ca0052 @724 Don't wait for async requests to complete automatically in close.
new d1210c4 @725 Tighten.
new 5b59829 @726 Only track ASYNC requests!
new 6ba3eb8 @727 You still need this - if someone calls and no outstanding.
new 82ec05b @728 Work out the async tracker.
new 9835a12 @729 Change order.
new 4a305cf @730 Put this test on disk.
new 65a3657 @731 Fix wait on close.
new 5ced9d4 @733 Complete on send exception.
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../stream/AnalyticsShardRequestManager.java | 5 +-
.../java/org/apache/solr/handler/IndexFetcher.java | 13 +--
.../handler/component/IterativeMergeStrategy.java | 13 +--
.../java/org/apache/solr/update/UpdateHandler.java | 12 +--
.../src/java/org/apache/solr/update/UpdateLog.java | 52 ++++-----
.../solr/client/solrj/impl/Http2SolrClient.java | 116 +++++++++++----------
.../client/solrj/io/stream/ExecutorStream.java | 8 +-
.../solrj/impl/CloudHttp2SolrClientTest.java | 1 +
...ntUpdateHttp2SolrClientMultiCollectionTest.java | 1 -
9 files changed, 115 insertions(+), 106 deletions(-)
[lucene-solr] 04/13: @723 Remove semi hack,
should not be needed now that we don't miss registering a queued
listener.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 072951f4637da009d45b290240fed5e0d38e30d4
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 12:10:09 2020 -0500
@723 Remove semi hack, should not be needed now that we don't miss registering a queued listener.
---
.../src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 32b95bd..ade6175 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -904,9 +904,7 @@ public class Http2SolrClient extends SolrClient {
public void waitForComplete() {
if (log.isDebugEnabled()) log.debug("Before wait for outstanding requests registered: {} arrived: {}", phaser.getRegisteredParties(), phaser.getArrivedParties());
- if (phaser.getUnarrivedParties() <= 1) {
- return;
- }
+
int arrival = phaser.arriveAndAwaitAdvance();
// phaser.awaitAdvance(phaser.arriveAndDeregister());
[lucene-solr] 13/13: @733 Complete on send exception.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 5ced9d4e1f21f3ec40decd9372ebc3ee3f0cb2c6
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 18:45:39 2020 -0500
@733 Complete on send exception.
---
.../solr/client/solrj/impl/Http2SolrClient.java | 48 ++++++++++++----------
1 file changed, 26 insertions(+), 22 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index f9a48f6..f95c158 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -398,32 +398,36 @@ public class Http2SolrClient extends SolrClient {
if (onComplete != null) {
asyncTracker.register();
// This async call only suitable for indexing since the response size is limited by 5MB
- req.send(new BufferingResponseListener(5 * 1024 * 1024) {
-
- @Override
- public void onComplete(Result result) {
- try {
- if (result.isFailed()) {
- onComplete.onFailure(result.getFailure());
- return;
- }
+ try {
+ req.send(new BufferingResponseListener(5 * 1024 * 1024) {
- NamedList<Object> rsp;
+ @Override
+ public void onComplete(Result result) {
try {
- InputStream is = getContentAsInputStream();
- assert ObjectReleaseTracker.track(is);
- rsp = processErrorsAndResponse(req, result.getResponse(),
- parser, is, getMediaType(), getEncoding(), isV2ApiRequest(solrRequest));
- onComplete.onSuccess(rsp);
- } catch (Exception e) {
- ParWork.propegateInterrupt(e);
+ if (result.isFailed()) {
+ onComplete.onFailure(result.getFailure());
+ return;
+ }
+
+ NamedList<Object> rsp;
+ try {
+ InputStream is = getContentAsInputStream();
+ assert ObjectReleaseTracker.track(is);
+ rsp = processErrorsAndResponse(req, result.getResponse(), parser, is, getMediaType(), getEncoding(), isV2ApiRequest(solrRequest));
+ onComplete.onSuccess(rsp);
+ } catch (Exception e) {
+ ParWork.propegateInterrupt(e);
+ }
+ } finally {
+ asyncTracker.completeListener.onComplete(result);
}
- } finally {
- asyncTracker.completeListener.onComplete(result);
}
- }
- });
- return null;
+ });
+ return null;
+ } catch (Exception e) {
+ onComplete.onFailure(e);
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, e);
+ }
} else {
try {
InputStreamResponseListener listener = new MyInputStreamResponseListener(asyncTracker);
[lucene-solr] 02/13: @721 Convert some more executor usage to
shared execs.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 7ffb68c0bb6f17b83908304bbc9f2bd8037f0f55
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 11:08:31 2020 -0500
@721 Convert some more executor usage to shared execs.
---
.../solr/analytics/stream/AnalyticsShardRequestManager.java | 5 +++--
.../core/src/java/org/apache/solr/handler/IndexFetcher.java | 13 +++++++------
.../solr/handler/component/IterativeMergeStrategy.java | 13 +++++++------
3 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/stream/AnalyticsShardRequestManager.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/stream/AnalyticsShardRequestManager.java
index 4ce5eb6..12e7b2c 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/stream/AnalyticsShardRequestManager.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/stream/AnalyticsShardRequestManager.java
@@ -37,6 +37,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
@@ -142,7 +143,7 @@ public class AnalyticsShardRequestManager {
* @throws IOException if an exception occurs while sending requests.
*/
private void streamFromShards() throws IOException {
- ExecutorService service = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("SolrAnalyticsStream"));
+ ExecutorService service = ParWork.getRootSharedExecutor();
List<Future<SolrException>> futures = new ArrayList<>();
List<AnalyticsShardRequester> openers = new ArrayList<>();
for (String replicaUrl : replicaUrls) {
@@ -163,11 +164,11 @@ public class AnalyticsShardRequestManager {
}
}
} catch (InterruptedException e1) {
+ ParWork.propegateInterrupt(e1);
throw new RuntimeException(e1);
} catch (ExecutionException e1) {
throw new RuntimeException(e1);
} finally {
- service.shutdown();
for (AnalyticsShardRequester opener : openers) {
opener.close();
}
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 93d7eeb..fcf59a7 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -154,6 +154,8 @@ public class IndexFetcher {
private volatile ExecutorService fsyncService;
+ private volatile Future<?> fsyncServiceFuture;
+
private volatile boolean stop = false;
private boolean useInternalCompression = false;
@@ -515,7 +517,7 @@ public class IndexFetcher {
}
// Create the sync service
- fsyncService = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrNamedThreadFactory("fsyncService"));
+ fsyncService = ParWork.getRootSharedExecutor();
// use a synchronized list because the list is read by other threads (to show details)
filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
// if the generation of master is older than that of the slave , it means they are not compatible to be copied
@@ -750,8 +752,8 @@ public class IndexFetcher {
markReplicationStop();
dirFileFetcher = null;
localFileFetcher = null;
- if (fsyncService != null && !fsyncService.isShutdown()) fsyncService.shutdown();
fsyncService = null;
+ fsyncServiceFuture = null;
stop = false;
fsyncException = null;
} finally {
@@ -809,10 +811,9 @@ public class IndexFetcher {
* terminate the fsync service and wait for all the tasks to complete. If it is already terminated
*/
private void terminateAndWaitFsyncService() throws Exception {
- if (fsyncService.isTerminated()) return;
- fsyncService.shutdown();
+ if (fsyncServiceFuture == null) return;
// give a long wait say 1 hr
- fsyncService.awaitTermination(3600, TimeUnit.SECONDS);
+ fsyncServiceFuture.get(3600, TimeUnit.SECONDS);
// if any fsync failed, throw that exception back
Exception fsyncExceptionCopy = fsyncException;
if (fsyncExceptionCopy != null) throw fsyncExceptionCopy;
@@ -1742,7 +1743,7 @@ public class IndexFetcher {
} finally {
cleanup();
//if cleanup succeeds . The file is downloaded fully. do an fsync
- fsyncService.submit(() -> {
+ fsyncServiceFuture = fsyncService.submit(() -> {
try {
file.sync();
} catch (IOException e) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java b/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java
index 08fc7fe..698e7d9 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java
@@ -25,6 +25,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -32,6 +33,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@@ -51,7 +53,7 @@ public abstract class IterativeMergeStrategy implements MergeStrategy {
public void merge(ResponseBuilder rb, ShardRequest sreq) {
rb._responseDocs = new SolrDocumentList(); // Null pointers will occur otherwise.
rb.onePassDistributedQuery = true; // Turn off the second pass distributed.
- executorService = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("IterativeMergeStrategy"));
+ executorService = ParWork.getRootSharedExecutor();
httpClient = getHttpClient();
try {
process(rb, sreq);
@@ -59,7 +61,6 @@ public abstract class IterativeMergeStrategy implements MergeStrategy {
throw new RuntimeException(e);
} finally {
HttpClientUtil.close(httpClient);
- executorService.shutdownNow();
}
}
@@ -79,13 +80,13 @@ public abstract class IterativeMergeStrategy implements MergeStrategy {
}
- public class CallBack implements Callable<CallBack> {
+ public static class CallBack implements Callable<CallBack> {
private HttpSolrClient solrClient;
private QueryRequest req;
private QueryResponse response;
private ShardResponse originalShardResponse;
- public CallBack(ShardResponse originalShardResponse, QueryRequest req) {
+ public CallBack(ShardResponse originalShardResponse, QueryRequest req, HttpClient httpClient) {
this.solrClient = new Builder(originalShardResponse.getShardAddress())
.withHttpClient(httpClient)
@@ -116,13 +117,13 @@ public abstract class IterativeMergeStrategy implements MergeStrategy {
@SuppressWarnings({"unchecked", "rawtypes"})
List<Future<CallBack>> futures = new ArrayList();
for(ShardResponse response : responses) {
- futures.add(this.executorService.submit(new CallBack(response, req)));
+ futures.add(this.executorService.submit(new CallBack(response, req, httpClient)));
}
return futures;
}
public Future<CallBack> callBack(ShardResponse response, QueryRequest req) {
- return this.executorService.submit(new CallBack(response, req));
+ return this.executorService.submit(new CallBack(response, req, httpClient));
}
protected abstract void process(ResponseBuilder rb, ShardRequest sreq) throws Exception;
[lucene-solr] 11/13: @730 Put this test on disk.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 4a305cf348a2a6ae8cf9bfd869912a6c4a6e8a4f
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 18:02:24 2020 -0500
@730 Put this test on disk.
---
.../test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java
index e602938..4925fbb 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java
@@ -104,6 +104,7 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
+ useFactory(null);
configureCluster(NODE_COUNT)
.addConfig(TEST_CONFIGSET_NAME, getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
.configure();
[lucene-solr] 10/13: @729 Change order.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 9835a126fdc616e32efd71887d32fa6d817c313e
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 16:44:04 2020 -0500
@729 Change order.
---
.../src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 628820c..f9a48f6 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -856,7 +856,7 @@ public class Http2SolrClient extends SolrClient {
// wait for async requests
private final Phaser phaser = new ThePhaser(1);
// maximum outstanding requests left
- private final Request.QueuedListener queuedListener;
+ private final Request.BeginListener queuedListener;
private final Response.CompleteListener completeListener;
AsyncTracker() {
@@ -906,12 +906,12 @@ public class Http2SolrClient extends SolrClient {
if (log.isDebugEnabled()) {
log.debug("Registered new party");
}
- phaser.register();
try {
available.acquire();
} catch (InterruptedException ignored) {
ParWork.propegateInterrupt(ignored);
}
+ phaser.register();
}
private static class ThePhaser extends Phaser {
[lucene-solr] 06/13: @725 Tighten.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit d1210c49882f2721f8e75f6912477ec425152a31
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 12:41:58 2020 -0500
@725 Tighten.
---
solr/core/src/java/org/apache/solr/update/UpdateHandler.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
index 6dde152..0baa28e 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
@@ -163,9 +163,12 @@ UpdateHandler implements SolrInfoBean, Closeable {
} else {
ourUpdateLog = updateLog;
}
- } catch (Exception e) {
+ } catch (Throwable e) {
IOUtils.closeQuietly(ourUpdateLog);
ObjectReleaseTracker.release(this);
+ if (e instanceof Error) {
+ throw e;
+ }
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
ulog = ourUpdateLog;
[lucene-solr] 05/13: @724 Don't wait for async requests to complete
automatically in close.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 4ca005296a691ac902449fc6c21f1b58d3290fc3
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 12:33:16 2020 -0500
@724 Don't wait for async requests to complete automatically in close.
---
.../src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index ade6175..b827705 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -252,7 +252,6 @@ public class Http2SolrClient extends SolrClient {
public void close() {
// closeTracker.close();
- asyncTracker.waitForComplete();
if (closeClient) {
try {
httpClient.stop();
[lucene-solr] 07/13: @726 Only track ASYNC requests!
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 5b598296713f65c4ae75af975ef57d7161be5635
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 13:07:42 2020 -0500
@726 Only track ASYNC requests!
---
.../apache/solr/client/solrj/impl/Http2SolrClient.java | 16 ++++------------
1 file changed, 4 insertions(+), 12 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index b827705..0c7f803 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -354,15 +354,11 @@ public class Http2SolrClient extends SolrClient {
InputStreamResponseListener responseListener = new InputStreamResponseListener() {
@Override
public void onComplete(Result result) {
- try {
- super.onComplete(result);
- } finally {
- asyncTracker.completeListener.onComplete(result);
- }
+ super.onComplete(result);
}
};
asyncTracker.register();
- postRequest.onRequestQueued(asyncTracker.queuedListener).send(responseListener);
+ postRequest.send(responseListener);
boolean isXml = ClientUtils.TEXT_XML.equals(requestWriter.getUpdateContentType());
OutStream outStream = new OutStream(collection, origParams, provider, responseListener,
@@ -438,14 +434,10 @@ public class Http2SolrClient extends SolrClient {
InputStreamResponseListener listener = new InputStreamResponseListener() {
@Override
public void onComplete(Result result) {
- try {
- super.onComplete(result);
- } finally {
- asyncTracker.completeListener.onComplete(result);
- }
+ super.onComplete(result);
}
};
- req.onRequestQueued(asyncTracker.queuedListener).send(listener);
+ req.send(listener);
Response response = listener.get(idleTimeout, TimeUnit.MILLISECONDS);
InputStream is = listener.getInputStream();
// nocommit - track this again when streaming use is fixed
[lucene-solr] 09/13: @728 Work out the async tracker.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 82ec05b3a21b15294342c6f6e2b19a3757d4b4f8
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 16:34:00 2020 -0500
@728 Work out the async tracker.
# Conflicts:
# solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientMultiCollectionTest.java
---
.../solr/client/solrj/impl/Http2SolrClient.java | 59 +++++++++++++---------
...ntUpdateHttp2SolrClientMultiCollectionTest.java | 1 -
2 files changed, 35 insertions(+), 25 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 0f0f536..628820c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -351,13 +351,8 @@ public class Http2SolrClient extends SolrClient {
}
decorateRequest(postRequest, updateRequest);
- InputStreamResponseListener responseListener = new InputStreamResponseListener() {
- @Override
- public void onComplete(Result result) {
- super.onComplete(result);
- }
- };
- asyncTracker.register();
+ InputStreamResponseListener responseListener = new OurInputStreamResponseListener();
+
postRequest.send(responseListener);
boolean isXml = ClientUtils.TEXT_XML.equals(requestWriter.getUpdateContentType());
@@ -401,9 +396,9 @@ public class Http2SolrClient extends SolrClient {
final ResponseParser parser = solrRequest.getResponseParser() == null
? this.parser: solrRequest.getResponseParser();
if (onComplete != null) {
+ asyncTracker.register();
// This async call only suitable for indexing since the response size is limited by 5MB
- req.onRequestQueued(asyncTracker.queuedListener)
- .send(new BufferingResponseListener(5 * 1024 * 1024) {
+ req.send(new BufferingResponseListener(5 * 1024 * 1024) {
@Override
public void onComplete(Result result) {
@@ -431,12 +426,7 @@ public class Http2SolrClient extends SolrClient {
return null;
} else {
try {
- InputStreamResponseListener listener = new InputStreamResponseListener() {
- @Override
- public void onComplete(Result result) {
- super.onComplete(result);
- }
- };
+ InputStreamResponseListener listener = new MyInputStreamResponseListener(asyncTracker);
req.send(listener);
Response response = listener.get(idleTimeout, TimeUnit.MILLISECONDS);
InputStream is = listener.getInputStream();
@@ -895,9 +885,9 @@ public class Http2SolrClient extends SolrClient {
public void waitForComplete() {
if (log.isDebugEnabled()) log.debug("Before wait for outstanding requests registered: {} arrived: {}", phaser.getRegisteredParties(), phaser.getArrivedParties());
- if (phaser.getUnarrivedParties() <= 1) {
- return;
- }
+// if (phaser.getUnarrivedParties() <= 1) {
+// return;
+// }
int arrival = phaser.arriveAndAwaitAdvance();
// phaser.awaitAdvance(phaser.arriveAndDeregister());
@@ -916,12 +906,12 @@ public class Http2SolrClient extends SolrClient {
if (log.isDebugEnabled()) {
log.debug("Registered new party");
}
- // phaser.register();
-// try {
-// available.acquire();
-// } catch (InterruptedException ignored) {
-// ParWork.propegateInterrupt(ignored);
-// }
+ phaser.register();
+ try {
+ available.acquire();
+ } catch (InterruptedException ignored) {
+ ParWork.propegateInterrupt(ignored);
+ }
}
private static class ThePhaser extends Phaser {
@@ -1238,4 +1228,25 @@ public class Http2SolrClient extends SolrClient {
ContentResponse response = httpClient.newRequest(url).method(PUT).content(new BytesContentProvider(bytes), contentType).send();
return response.getContentAsString();
}
+
+ private static class MyInputStreamResponseListener extends InputStreamResponseListener {
+
+ AsyncTracker tracker;
+
+ MyInputStreamResponseListener(AsyncTracker tracker) {
+ this.tracker = tracker;
+ }
+
+ @Override
+ public void onComplete(Result result) {
+ super.onComplete(result);
+ }
+ }
+
+ private static class OurInputStreamResponseListener extends InputStreamResponseListener {
+ @Override
+ public void onComplete(Result result) {
+ super.onComplete(result);
+ }
+ }
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientMultiCollectionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientMultiCollectionTest.java
index 1600882..71ae4d3 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientMultiCollectionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientMultiCollectionTest.java
@@ -36,7 +36,6 @@ import org.junit.Test;
* {@link ConcurrentUpdateSolrClient} reuses the same HTTP connection to send multiple requests. These tests ensure
* that this connection-reuse never results in documents being sent to the wrong collection. See SOLR-12803
*/
-@Ignore // nocommit debug
public class ConcurrentUpdateHttp2SolrClientMultiCollectionTest extends SolrCloudTestCase {
private static final String COLLECTION_ONE_NAME = "collection1";
[lucene-solr] 03/13: @722 Try and prevent UpdateLog leak.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit c22e43ec8f88eece9548ec01941a0076a8d818fc
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 11:56:10 2020 -0500
@722 Try and prevent UpdateLog leak.
---
solr/core/src/java/org/apache/solr/update/UpdateHandler.java | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
index 95dc7e4..6dde152 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
@@ -160,9 +160,6 @@ UpdateHandler implements SolrInfoBean, Closeable {
}
ourUpdateLog.init(ulogPluginInfo);
ourUpdateLog.init(this, core);
- if (updateLog != null) {
- updateLog.close();
- }
} else {
ourUpdateLog = updateLog;
}
@@ -170,9 +167,9 @@ UpdateHandler implements SolrInfoBean, Closeable {
IOUtils.closeQuietly(ourUpdateLog);
ObjectReleaseTracker.release(this);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } finally {
+ ulog = ourUpdateLog;
}
-
- ulog = ourUpdateLog;
}
/**
[lucene-solr] 08/13: @727 You still need this - if someone calls
and no outstanding.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 6ba3eb88f5d775d948ee584bf06b9ab50ba4ae55
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 13:35:22 2020 -0500
@727 You still need this - if someone calls and no outstanding.
---
.../src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 0c7f803..0f0f536 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -895,7 +895,9 @@ public class Http2SolrClient extends SolrClient {
public void waitForComplete() {
if (log.isDebugEnabled()) log.debug("Before wait for outstanding requests registered: {} arrived: {}", phaser.getRegisteredParties(), phaser.getArrivedParties());
-
+ if (phaser.getUnarrivedParties() <= 1) {
+ return;
+ }
int arrival = phaser.arriveAndAwaitAdvance();
// phaser.awaitAdvance(phaser.arriveAndDeregister());
[lucene-solr] 01/13: @720 UpdateLog executor usage fix/improve.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 7f63eb81e020c775442a99d5dca1d977f0c52458
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 10:20:12 2020 -0500
@720 UpdateLog executor usage fix/improve.
---
.../src/java/org/apache/solr/update/UpdateLog.java | 52 ++++++++++------------
1 file changed, 23 insertions(+), 29 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index a9e319e..e5c9069 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -553,12 +553,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
public String[] getLogList(File directory) {
final String prefix = TLOG_NAME+'.';
- String[] names = directory.list(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith(prefix);
- }
- });
+ String[] names = directory.list(new MyFilenameFilter(prefix));
if (names == null) {
throw new RuntimeException(new FileNotFoundException(directory.getAbsolutePath()));
}
@@ -1010,7 +1005,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
// This can happen when trying to deserialize the entry at position lookupPointer,
// but from a different tlog than the one containing the desired entry.
// Just ignore the exception, so as to proceed to the next tlog.
- log.debug("Exception reading the log (this is expected, don't worry)={}, for version={}. This can be ignored"
+ log.info("Exception reading the log (this is expected, don't worry)={}, for version={}. This can be ignored"
, lookupLog, lookupVersion);
}
@@ -1165,7 +1160,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
if (recoverLogs.isEmpty()) return null;
- ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(recoveryExecutor);
+ ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(ParWork.getRootSharedExecutor());
LogReplayer replayer = new LogReplayer(recoverLogs, false);
versionInfo.blockUpdates();
@@ -1201,7 +1196,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
recoveryInfo = new RecoveryInfo();
tlog.incref();
- ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(recoveryExecutor);
+ ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(ParWork.getRootSharedExecutor());
LogReplayer replayer = new LogReplayer(Collections.singletonList(tlog), false, true);
versionInfo.blockUpdates();
@@ -1403,7 +1398,6 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
public void close(boolean committed, boolean deleteOnClose) {
this.isClosed = true;
- recoveryExecutor.shutdown(); // no new tasks
synchronized (this) {
@@ -1428,13 +1422,6 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
- try {
- ExecutorUtil.shutdownAndAwaitTermination(recoveryExecutor);
- } catch (Exception e) {
- ParWork.propegateInterrupt(e);
- SolrException.log(log, e);
- }
-
ObjectReleaseTracker.release(this);
}
@@ -1456,17 +1443,32 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
}
- public class RecentUpdates implements Closeable {
+ private static class MyFilenameFilter implements FilenameFilter {
+ private final String prefix;
+
+ public MyFilenameFilter(String prefix) {
+ this.prefix = prefix;
+ }
+
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.startsWith(prefix);
+ }
+ }
+
+ public static class RecentUpdates implements Closeable {
final Deque<TransactionLog> logList; // newest first
+ private final int numRecordsToKeep;
List<List<Update>> updateList;
HashMap<Long, Update> updates;
List<Update> deleteByQueryList;
List<DeleteUpdate> deleteList;
Set<Long> bufferUpdates = new HashSet<>();
- public RecentUpdates(Deque<TransactionLog> logList) {
+ public RecentUpdates(Deque<TransactionLog> logList, int numRecordsToKeep) {
this.logList = logList;
+ this.numRecordsToKeep = numRecordsToKeep;
boolean success = false;
try {
update();
@@ -1648,7 +1650,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
// TODO: what if I hand out a list of updates, then do an update, then hand out another list (and
// one of the updates I originally handed out fell off the list). Over-request?
- return new RecentUpdates(logList);
+ return new RecentUpdates(logList, numRecordsToKeep);
}
@@ -1737,10 +1739,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
versionInfo.unblockUpdates();
}
- if (recoveryExecutor.isShutdown()) {
- throw new RuntimeException("executor is not running...");
- }
- ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(recoveryExecutor);
+ ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(ParWork.getRootSharedExecutor());
LogReplayer replayer = new LogReplayer(Collections.singletonList(bufferTlog), true);
return cs.submit(() -> {
replayer.run();
@@ -2144,11 +2143,6 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
return cmd;
}
-
- ThreadPoolExecutor recoveryExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0,
- Integer.MAX_VALUE, 1, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new SolrNamedThreadFactory("recoveryExecutor"));
-
public static void deleteFile(File file) {
boolean success = false;
[lucene-solr] 12/13: @731 Fix wait on close.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 65a3657f679dff159423ada407963eaca8fb5ae3
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Sep 3 18:25:41 2020 -0500
@731 Fix wait on close.
---
.../org/apache/solr/client/solrj/io/stream/ExecutorStream.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExecutorStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExecutorStream.java
index 740ee75..8d67b0a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExecutorStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ExecutorStream.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Locale;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
@@ -132,11 +133,16 @@ public class ExecutorStream extends TupleStream implements Expressible {
}
public void open() throws IOException {
- executorService = ParWork.getRootSharedExecutor();
+ executorService = ParWork.getMyPerThreadExecutor();
stream.open();
}
public void close() throws IOException {
+ try {
+ executorService.awaitTermination(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ ParWork.propegateInterrupt(e);
+ }
stream.close();
}