You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/11/10 22:10:28 UTC
[lucene-solr] branch reference_impl_dev updated (e3f0250 -> 1af452d)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from e3f0250 @1142 Don't hold zkwriter lock while waiting for updates to propagate.
new 287c94e @1143 Tweak http client settings.
new 5e11514 @1144 Tweaking zkstatewriter towards right.
new 711be62 @1145 Raise timeouts for prod.
new ea5efd7 @1146 should not be replaying tlogs
new 376d236 @1147 Don't use version from clusterstate object.
new de14012 @1148 Bump limits again.
new 1af452d @1149 Whoops, need stronger sync.
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/solr/cloud/ZkController.java | 4 +-
.../apache/solr/cloud/overseer/ZkStateWriter.java | 214 +++++++++++----------
.../apache/solr/update/DirectUpdateHandler2.java | 4 +-
.../solr/client/solrj/impl/Http2SolrClient.java | 10 +-
.../org/apache/solr/common/cloud/ClusterState.java | 2 +-
.../apache/solr/common/cloud/ZkStateReader.java | 59 +++---
6 files changed, 155 insertions(+), 138 deletions(-)
[lucene-solr] 04/07: @1146 should not be replaying tlogs
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ea5efd757e90ae5ab724fa3afb99ccb9a414e431
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 15:28:46 2020 -0600
@1146 should not be replaying tlogs
---
solr/core/src/java/org/apache/solr/cloud/ZkController.java | 2 +-
solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index fc57248..3a8b589 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1357,7 +1357,7 @@ public class ZkController implements Closeable, Runnable {
log.info("Register SolrCore, core={} baseUrl={} collection={}, shard={} skipRecovery={}", coreName, baseUrl, collection, shardId, skipRecovery);
AtomicReference<DocCollection> coll = new AtomicReference<>();
try {
- zkStateReader.waitForState(collection, 30, TimeUnit.SECONDS, (l, c) -> { // nocommit timeout
+ zkStateReader.waitForState(collection, 60, TimeUnit.SECONDS, (l, c) -> { // nocommit timeout
// if (isClosed()) {
// throw new AlreadyClosedException();
// }
diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
index fb14c57..af952f2 100644
--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
@@ -842,7 +842,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
}
// we shouldn't close the transaction logs either, but leaving them open
// means we can't delete them on windows (needed for tests)
- if (ulog != null) ulog.close(false);
+ if (ulog != null) ulog.close(true);
return;
}
@@ -896,7 +896,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
}
try {
- if (ulog != null) ulog.close(false);
+ if (ulog != null) ulog.close(true);
} catch (Throwable th) {
log.error("Error closing log files", th);
if (th instanceof OutOfMemoryError) {
[lucene-solr] 02/07: @1144 Tweaking zkstatewriter towards right.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 5e11514801871b615dc4c50d21f2c477f666b6c3
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 14:58:16 2020 -0600
@1144 Tweaking zkstatewriter towards right.
---
.../org/apache/solr/cloud/overseer/ZkStateWriter.java | 15 ++++++++++++---
.../java/org/apache/solr/common/cloud/ClusterState.java | 2 +-
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index d81d122..512183a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
@@ -255,6 +256,7 @@ public class ZkStateWriter {
public void writePendingUpdates() {
writeLock.lock();
+ Set<String> collectionsToWaitFor = ConcurrentHashMap.newKeySet();
try {
ourLock.lock();
try {
@@ -271,6 +273,12 @@ public class ZkStateWriter {
failedUpdates.clear();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, lastFailedException.get());
}
+
+ for (String collectionName : cs.getCollectionStates().keySet()) {
+ DocCollection collection = cs.getCollectionOrNull(collectionName);
+ if (collection == null) return;
+ collectionsToWaitFor.add(collection.getName());
+ }
} finally {
ourLock.unlock();
}
@@ -313,6 +321,7 @@ public class ZkStateWriter {
});
ourLock.lock();
+ AtomicInteger lastVersion = new AtomicInteger();
try {
cs.forEachCollection(collection -> {
if (collectionsToWrite.contains(collection.getName())) {
@@ -333,8 +342,8 @@ public class ZkStateWriter {
if (v != null) {
version = v;
}
-
- reader.getZkClient().setData(path, data, version == 0 ? -1 : version, true);
+ lastVersion.set(version);
+ reader.getZkClient().setData(path, data, version, true);
trackVersions.put(collection.getName(), version + 1);
} catch (KeeperException.NoNodeException e) {
@@ -346,7 +355,7 @@ public class ZkStateWriter {
failedUpdates.put(collection.getName(), collection);
stat = reader.getZkClient().exists(path, null);
// this is a tragic error, we must disallow usage of this instance
- log.warn("Tried to update the cluster state using version={} but we where rejected, found {}", collection.getZNodeVersion(), stat.getVersion(), bve);
+ log.warn("Tried to update the cluster state using version={} but we where rejected, found {}", lastVersion.get(), stat.getVersion(), bve);
}
if (log.isDebugEnabled()) log.debug("Set version for local collection {} to {}", collection.getName(), collection.getZNodeVersion() + 1);
} catch (InterruptedException | AlreadyClosedException e) {
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index 27cb7b6..1653e51 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -356,7 +356,7 @@ public class ClusterState implements JSONWriter.Writable {
* CollectionRef.get() != null
*/
public Map<String, CollectionRef> getCollectionStates() {
- return immutableCollectionStates;
+ return collectionStates;
}
/**
[lucene-solr] 03/07: @1145 Raise timeouts for prod.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 711be6208c4442db7f2f97c0367ffde35fd3e21c
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 15:14:39 2020 -0600
@1145 Raise timeouts for prod.
---
solr/core/src/java/org/apache/solr/cloud/ZkController.java | 4 ++--
.../src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java | 9 +--------
2 files changed, 3 insertions(+), 10 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 10a86d4..fc57248 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1357,7 +1357,7 @@ public class ZkController implements Closeable, Runnable {
log.info("Register SolrCore, core={} baseUrl={} collection={}, shard={} skipRecovery={}", coreName, baseUrl, collection, shardId, skipRecovery);
AtomicReference<DocCollection> coll = new AtomicReference<>();
try {
- zkStateReader.waitForState(collection, 3, TimeUnit.SECONDS, (l, c) -> { // nocommit timeout
+ zkStateReader.waitForState(collection, 30, TimeUnit.SECONDS, (l, c) -> { // nocommit timeout
// if (isClosed()) {
// throw new AlreadyClosedException();
// }
@@ -1410,7 +1410,7 @@ public class ZkController implements Closeable, Runnable {
}
log.info("Wait to see leader for {}, {}", collection, shardId);
- Replica leader = zkStateReader.getLeaderRetry(collection, shardId, 10000);
+ Replica leader = zkStateReader.getLeaderRetry(collection, shardId, 15000);
String ourUrl = replica.getCoreUrl();
boolean isLeader = leader.getName() .equals(coreName);
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 512183a..c90db99 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -256,7 +256,6 @@ public class ZkStateWriter {
public void writePendingUpdates() {
writeLock.lock();
- Set<String> collectionsToWaitFor = ConcurrentHashMap.newKeySet();
try {
ourLock.lock();
try {
@@ -273,17 +272,11 @@ public class ZkStateWriter {
failedUpdates.clear();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, lastFailedException.get());
}
-
- for (String collectionName : cs.getCollectionStates().keySet()) {
- DocCollection collection = cs.getCollectionOrNull(collectionName);
- if (collection == null) return;
- collectionsToWaitFor.add(collection.getName());
- }
} finally {
ourLock.unlock();
}
- // wait to see our last publish version has propagated
+ // wait to see our last publish version has propagated TODO don't wait on collections not hosted on overseer?
cs.forEachCollection(collection -> {
if (collectionsToWrite.contains(collection.getName())) {
Integer v = null;
[lucene-solr] 05/07: @1147 Don't use version from clusterstate
object.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 376d23659eebfd636d8e1614e9b6162e77bbb0d6
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 15:40:35 2020 -0600
@1147 Don't use version from clusterstate object.
---
solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index c90db99..971e505 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -330,10 +330,13 @@ public class ZkStateWriter {
if (log.isDebugEnabled()) log.debug("Write state.json prevVersion={} bytes={} col={}", collection.getZNodeVersion(), data.length, collection);
try {
- int version = collection.getZNodeVersion();
+
Integer v = trackVersions.get(collection.getName());
+ Integer version;
if (v != null) {
version = v;
+ } else {
+ version = reader.getZkClient().exists(path, null).getVersion();
}
lastVersion.set(version);
reader.getZkClient().setData(path, data, version, true);
[lucene-solr] 01/07: @1143 Tweak http client settings.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 287c94ebda9d9ca154e26bfd77026b914a398a7d
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 14:18:16 2020 -0600
@1143 Tweak http client settings.
---
.../apache/solr/cloud/overseer/ZkStateWriter.java | 209 +++++++++++----------
.../solr/client/solrj/impl/Http2SolrClient.java | 10 +-
2 files changed, 113 insertions(+), 106 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index f9268c5..d81d122 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -68,6 +68,7 @@ public class ZkStateWriter {
private Set<String> collectionsToWrite = new HashSet<>();
protected final ReentrantLock ourLock = new ReentrantLock(true);
+ protected final ReentrantLock writeLock = new ReentrantLock(true);
public ZkStateWriter(ZkStateReader zkStateReader, Stats stats) {
assert zkStateReader != null;
@@ -252,116 +253,122 @@ public class ZkStateWriter {
*
*/
public void writePendingUpdates() {
- ourLock.lock();
+
+ writeLock.lock();
try {
- if (!dirty) {
- return;
- }
+ ourLock.lock();
+ try {
+ if (!dirty) {
+ return;
+ }
- if (log.isDebugEnabled()) {
- log.debug("writePendingUpdates {}", cs);
- }
+ if (log.isDebugEnabled()) {
+ log.debug("writePendingUpdates {}", cs);
+ }
- if (failedUpdates.size() > 0) {
- log.warn("Some collection updates failed {} logging last exception", failedUpdates, lastFailedException); // nocommit expand
- failedUpdates.clear();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, lastFailedException.get());
+ if (failedUpdates.size() > 0) {
+ log.warn("Some collection updates failed {} logging last exception", failedUpdates, lastFailedException); // nocommit expand
+ failedUpdates.clear();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, lastFailedException.get());
+ }
+ } finally {
+ ourLock.unlock();
}
- } finally {
- ourLock.unlock();
- }
- // wait to see our last publish version has propagated
- cs.forEachCollection(collection -> {
- if (collectionsToWrite.contains(collection.getName())) {
- Integer v = null;
- try {
- //System.out.println("waiting to see state " + prevVersion);
- v = trackVersions.get(collection.getName());
- if (v == null) v = 0;
- if (v == 0) return;
- Integer version = v;
+ // wait to see our last publish version has propagated
+ cs.forEachCollection(collection -> {
+ if (collectionsToWrite.contains(collection.getName())) {
+ Integer v = null;
try {
- log.debug("wait to see last published version for collection {} {}", collection.getName(), v);
- reader.waitForState(collection.getName(), 5, TimeUnit.SECONDS, (l, col) -> {
- if (col == null) {
- return true;
- }
- // if (col != null) {
- // log.info("the version " + col.getZNodeVersion());
- // }
- if (col != null && col.getZNodeVersion() >= version) {
- if (log.isDebugEnabled()) log.debug("Waited for ver: {}", col.getZNodeVersion() + 1);
- // System.out.println("found the version");
- return true;
- }
- return false;
- });
- } catch (InterruptedException e) {
- ParWork.propagateInterrupt(e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ //System.out.println("waiting to see state " + prevVersion);
+ v = trackVersions.get(collection.getName());
+ if (v == null) v = 0;
+ if (v == 0) return;
+ Integer version = v;
+ try {
+ log.debug("wait to see last published version for collection {} {}", collection.getName(), v);
+ reader.waitForState(collection.getName(), 5, TimeUnit.SECONDS, (l, col) -> {
+ if (col == null) {
+ return true;
+ }
+ // if (col != null) {
+ // log.info("the version " + col.getZNodeVersion());
+ // }
+ if (col != null && col.getZNodeVersion() >= version) {
+ if (log.isDebugEnabled()) log.debug("Waited for ver: {}", col.getZNodeVersion() + 1);
+ // System.out.println("found the version");
+ return true;
+ }
+ return false;
+ });
+ } catch (InterruptedException e) {
+ ParWork.propagateInterrupt(e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ } catch (TimeoutException e) {
+ log.warn("Timeout waiting to see written cluster state come back " + v);
}
- } catch (TimeoutException e) {
- log.warn("Timeout waiting to see written cluster state come back " + v);
}
- }
- });
-
- ourLock.lock();
- try {
- cs.forEachCollection(collection -> {
- if (collectionsToWrite.contains(collection.getName())) {
- String name = collection.getName();
- String path = ZkStateReader.getCollectionPath(collection.getName());
- if (log.isDebugEnabled()) log.debug("process {}", collection);
- Stat stat = new Stat();
- boolean success = false;
- try {
-
- byte[] data = Utils.toJSON(singletonMap(name, collection));
-
- if (log.isDebugEnabled()) log.debug("Write state.json prevVersion={} bytes={} col={}", collection.getZNodeVersion(), data.length, collection);
-
- try {
- int version = collection.getZNodeVersion();
- Integer v = trackVersions.get(collection.getName());
- if (v != null) {
- version = v;
- }
-
- reader.getZkClient().setData(path, data, version == 0 ? -1 : version, true);
-
- trackVersions.put(collection.getName(), version + 1);
- } catch (KeeperException.NoNodeException e) {
- if (log.isDebugEnabled()) log.debug("No node found for state.json", e);
- trackVersions.remove(collection.getName());
- // likely deleted
- } catch (KeeperException.BadVersionException bve) {
- lastFailedException.set(bve);
- failedUpdates.put(collection.getName(), collection);
- stat = reader.getZkClient().exists(path, null);
- // this is a tragic error, we must disallow usage of this instance
- log.warn("Tried to update the cluster state using version={} but we where rejected, found {}", collection.getZNodeVersion(), stat.getVersion(), bve);
- }
- if (log.isDebugEnabled()) log.debug("Set version for local collection {} to {}", collection.getName(), collection.getZNodeVersion() + 1);
- } catch (InterruptedException | AlreadyClosedException e) {
- log.info("We have been closed or one of our resources has, bailing {}", e.getClass().getSimpleName() + ":" + e.getMessage());
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } catch (KeeperException.SessionExpiredException e) {
- log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } catch (Exception e) {
- log.error("Failed processing update=" + collection, e);
- }
- }
- });
-
- dirty = false;
- collectionsToWrite.clear();
- } finally {
- ourLock.unlock();
- }
+ });
+
+ ourLock.lock();
+ try {
+ cs.forEachCollection(collection -> {
+ if (collectionsToWrite.contains(collection.getName())) {
+ String name = collection.getName();
+ String path = ZkStateReader.getCollectionPath(collection.getName());
+ if (log.isDebugEnabled()) log.debug("process {}", collection);
+ Stat stat = new Stat();
+ boolean success = false;
+ try {
+
+ byte[] data = Utils.toJSON(singletonMap(name, collection));
+
+ if (log.isDebugEnabled()) log.debug("Write state.json prevVersion={} bytes={} col={}", collection.getZNodeVersion(), data.length, collection);
+
+ try {
+ int version = collection.getZNodeVersion();
+ Integer v = trackVersions.get(collection.getName());
+ if (v != null) {
+ version = v;
+ }
+
+ reader.getZkClient().setData(path, data, version == 0 ? -1 : version, true);
+
+ trackVersions.put(collection.getName(), version + 1);
+ } catch (KeeperException.NoNodeException e) {
+ if (log.isDebugEnabled()) log.debug("No node found for state.json", e);
+ trackVersions.remove(collection.getName());
+ // likely deleted
+ } catch (KeeperException.BadVersionException bve) {
+ lastFailedException.set(bve);
+ failedUpdates.put(collection.getName(), collection);
+ stat = reader.getZkClient().exists(path, null);
+ // this is a tragic error, we must disallow usage of this instance
+ log.warn("Tried to update the cluster state using version={} but we where rejected, found {}", collection.getZNodeVersion(), stat.getVersion(), bve);
+ }
+ if (log.isDebugEnabled()) log.debug("Set version for local collection {} to {}", collection.getName(), collection.getZNodeVersion() + 1);
+ } catch (InterruptedException | AlreadyClosedException e) {
+ log.info("We have been closed or one of our resources has, bailing {}", e.getClass().getSimpleName() + ":" + e.getMessage());
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } catch (KeeperException.SessionExpiredException e) {
+ log.error("", e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } catch (Exception e) {
+ log.error("Failed processing update=" + collection, e);
+ }
+ }
+ });
+
+ dirty = false;
+ collectionsToWrite.clear();
+ } finally {
+ ourLock.unlock();
+ }
+ } finally {
+ writeLock.unlock();
+ }
// nocommit - harden against failures and exceptions
// if (log.isDebugEnabled()) {
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index aff1ad0..201fea1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -228,12 +228,12 @@ public class Http2SolrClient extends SolrClient {
} else {
log.info("Create Http2SolrClient with HTTP/1.1 transport");
}
- SolrHttpClientTransportOverHTTP transport = new SolrHttpClientTransportOverHTTP(2);
+ SolrHttpClientTransportOverHTTP transport = new SolrHttpClientTransportOverHTTP(4);
httpClient = new SolrInternalHttpClient(transport, sslContextFactory);
} else {
log.info("Create Http2SolrClient with HTTP/2 transport");
HTTP2Client http2client = new HTTP2Client();
- http2client.setSelectors(2);
+ http2client.setSelectors(4);
http2client.setMaxConcurrentPushedStreams(512);
http2client.setInputBufferSize(8192);
HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(http2client);
@@ -979,7 +979,7 @@ public class Http2SolrClient extends SolrClient {
public AsyncTracker(int maxOutstandingAsyncRequests) {
if (maxOutstandingAsyncRequests > 0) {
- available = new Semaphore(maxOutstandingAsyncRequests, true);
+ available = new Semaphore(maxOutstandingAsyncRequests, false);
} else {
available = null;
}
@@ -995,7 +995,7 @@ public class Http2SolrClient extends SolrClient {
public void close() {
phaser.forceTermination();
- if (available != null) available.release(available.getQueueLength() + 5);
+ if (available != null) available.drainPermits();
}
public void register() {
@@ -1038,7 +1038,7 @@ public class Http2SolrClient extends SolrClient {
public static class Builder {
- public int maxThreadPoolSize = Integer.getInteger("solr.maxHttp2ClientThreads", Math.max(64, ParWork.PROC_COUNT));
+ public int maxThreadPoolSize = Integer.getInteger("solr.maxHttp2ClientThreads", 128);
public int maxRequestsQueuedPerDestination = 2048;
private Http2SolrClient http2SolrClient;
private SSLConfig sslConfig = defaultSSLConfig;
[lucene-solr] 07/07: @1149 Whoops, need stronger sync.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1af452d8855716e4c5f54d82753523a6342ad016
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 16:10:06 2020 -0600
@1149 Whoops, need stronger sync.
---
.../apache/solr/common/cloud/ZkStateReader.java | 59 ++++++++++++----------
1 file changed, 32 insertions(+), 27 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 31cf7d6..88c962d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -1636,22 +1636,25 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
AtomicBoolean watchSet = new AtomicBoolean(false);
- collectionWatches.compute(collection, (k, v) -> {
- if (v == null) {
- v = new CollectionWatch<>();
- watchSet.set(true);
- }
- v.stateWatchers.add(stateWatcher);
- return v;
- });
+ synchronized (collectionWatches) {
- if (watchSet.get()) {
- new StateWatcher(collection).refreshAndWatch();
- }
+ collectionWatches.compute(collection, (k, v) -> {
+ if (v == null) {
+ v = new CollectionWatch<>();
+ watchSet.set(true);
+ }
+ v.stateWatchers.add(stateWatcher);
+ return v;
+ });
- DocCollection state = clusterState.getCollectionOrNull(collection);
- if (stateWatcher.onStateChanged(state) == true) {
- removeDocCollectionWatcher(collection, stateWatcher);
+ if (watchSet.get()) {
+ new StateWatcher(collection).refreshAndWatch();
+ }
+
+ DocCollection state = clusterState.getCollectionOrNull(collection);
+ if (stateWatcher.onStateChanged(state) == true) {
+ removeDocCollectionWatcher(collection, stateWatcher);
+ }
}
}
@@ -1866,19 +1869,21 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
}
AtomicBoolean reconstructState = new AtomicBoolean(false);
-
- collectionWatches.compute(collection, (k, v) -> {
- if (v == null) return null;
- v.stateWatchers.remove(watcher);
- if (v.canBeRemoved()) {
- log.info("no longer watch collection {}", collection);
- watchedCollectionStates.remove(collection);
- lazyCollectionStates.put(collection, new LazyCollectionRef(collection));
- reconstructState.set(true);
- return null;
- }
- return v;
- });
+ synchronized (collectionWatches) {
+
+ collectionWatches.compute(collection, (k, v) -> {
+ if (v == null) return null;
+ v.stateWatchers.remove(watcher);
+ if (v.canBeRemoved()) {
+ log.info("no longer watch collection {}", collection);
+ watchedCollectionStates.remove(collection);
+ lazyCollectionStates.put(collection, new LazyCollectionRef(collection));
+ reconstructState.set(true);
+ return null;
+ }
+ return v;
+ });
+ }
if (reconstructState.get()) {
updateLock.lock();
[lucene-solr] 06/07: @1148 Bump limits again.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit de140128d2efd84c40acd6e6fbff62e6eabd478f
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Nov 10 15:47:28 2020 -0600
@1148 Bump limits again.
---
.../src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 201fea1..2eab87a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -228,12 +228,12 @@ public class Http2SolrClient extends SolrClient {
} else {
log.info("Create Http2SolrClient with HTTP/1.1 transport");
}
- SolrHttpClientTransportOverHTTP transport = new SolrHttpClientTransportOverHTTP(4);
+ SolrHttpClientTransportOverHTTP transport = new SolrHttpClientTransportOverHTTP(6);
httpClient = new SolrInternalHttpClient(transport, sslContextFactory);
} else {
log.info("Create Http2SolrClient with HTTP/2 transport");
HTTP2Client http2client = new HTTP2Client();
- http2client.setSelectors(4);
+ http2client.setSelectors(6);
http2client.setMaxConcurrentPushedStreams(512);
http2client.setInputBufferSize(8192);
HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(http2client);
@@ -1038,7 +1038,7 @@ public class Http2SolrClient extends SolrClient {
public static class Builder {
- public int maxThreadPoolSize = Integer.getInteger("solr.maxHttp2ClientThreads", 128);
+ public int maxThreadPoolSize = Integer.getInteger("solr.maxHttp2ClientThreads", 512);
public int maxRequestsQueuedPerDestination = 2048;
private Http2SolrClient http2SolrClient;
private SSLConfig sslConfig = defaultSSLConfig;