You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/07/11 21:55:58 UTC
[lucene-solr] branch reference_impl updated: #76 More Tweaking on
thread management.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl by this push:
new 2bb277c #76 More Tweaking on thread management.
2bb277c is described below
commit 2bb277cdfe96b32977d7d4875fa01aa3a53d04ec
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Sat Jul 11 16:55:44 2020 -0500
#76 More Tweaking on thread management.
---
.../src/java/org/apache/solr/cloud/Overseer.java | 1 +
.../java/org/apache/solr/core/CoreContainer.java | 1 +
.../src/java/org/apache/solr/core/ZkContainer.java | 2 +-
.../solr/handler/admin/CoreAdminHandler.java | 3 +-
.../solr/handler/component/HttpShardHandler.java | 3 ++
.../java/org/apache/solr/request/SimpleFacets.java | 1 +
.../org/apache/solr/update/SolrCmdDistributor.java | 2 +-
.../src/java/org/apache/solr/common/ParWork.java | 57 ++++++++++++----------
.../org/apache/solr/SolrIgnoredThreadsFilter.java | 4 ++
9 files changed, 44 insertions(+), 30 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 1b7804a..f79ea87 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -416,6 +416,7 @@ public class Overseer implements SolrCloseable {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Message missing " + QUEUE_OPERATION + ":" + message);
}
AtomicBoolean stop = new AtomicBoolean(false);
+
ParWork.getExecutor().invokeAll(Collections.singleton(new Callable<Object>() { // ### expert use
@Override
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 69d1a73..c3feade 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -841,6 +841,7 @@ public class CoreContainer implements Closeable {
solrCores.markCoreAsLoading(cd);
}
if (cd.isLoadOnStartup()) {
+ ParWork.sizePoolByLoad();
futures.add(ParWork.getExecutor().submit(() -> {
SolrCore core;
try {
diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java b/solr/core/src/java/org/apache/solr/core/ZkContainer.java
index e2c331d..3b4881d 100644
--- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java
@@ -222,7 +222,7 @@ public class ZkContainer implements Closeable {
MDCLoggingContext.clear();
}
};
-
+ ParWork.sizePoolByLoad();
ParWork.getExecutor().submit(r); // ### expert usage
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index 77503ed..dc643b0 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -179,7 +179,8 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
try {
MDC.put("CoreAdminHandler.asyncId", taskId);
MDC.put("CoreAdminHandler.action", op.action.toString());
- ParWork.getExecutor().execute(() -> { // ### SUPER DUPER EXPERT USAGE
+ ParWork.sizePoolByLoad();
+ ParWork.getExecutor().submit(() -> { // ### SUPER DUPER EXPERT USAGE
boolean exceptionCaught = false;
try {
if (!cores.isShutDown()) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
index 17252c7..94ee216 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
@@ -30,6 +30,7 @@ import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.routing.ReplicaListTransformer;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -57,6 +58,7 @@ public class HttpShardHandler extends ShardHandler {
public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, Http2SolrClient httpClient) {
this.httpClient = httpClient;
this.httpShardHandlerFactory = httpShardHandlerFactory;
+
completionService = httpShardHandlerFactory.newCompletionService();
pending = new HashSet<>();
}
@@ -67,6 +69,7 @@ public class HttpShardHandler extends ShardHandler {
ShardRequestor shardRequestor = new ShardRequestor(sreq, shard, params, this);
try {
shardRequestor.init();
+ ParWork.sizePoolByLoad();
pending.add(completionService.submit(shardRequestor));
} finally {
shardRequestor.end();
diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
index 16d24c5..b04a4f5 100644
--- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
+++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
@@ -548,6 +548,7 @@ public class SimpleFacets {
} else {
PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset, limit, mincount, missing, sort, prefix, termFilter);
ps.setNumThreads(threads);
+ ParWork.sizePoolByLoad();
counts = ps.getFacetCounts(ParWork.getExecutor()); // ### expert usage
}
break;
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index f004262..920c145 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -298,7 +298,7 @@ public class SolrCmdDistributor implements Closeable {
if (SolrRequestInfo.getRequestInfo() != null) {
req.uReq.setUserPrincipal(SolrRequestInfo.getRequestInfo().getReq().getUserPrincipal());
}
-
+ ParWork.sizePoolByLoad();
Tracer tracer = GlobalTracer.getTracer();
Span parentSpan = tracer.activeSpan();
if (parentSpan != null) {
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWork.java b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
index a6c7265..97ef449 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWork.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
@@ -500,34 +500,8 @@ public class ParWork implements Closeable {
if (closeCalls.size() > 0) {
try {
- double load = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
- if (load < 0) {
- log.warn("SystemLoadAverage not supported on this JVM");
- load = 0;
- }
+ sizePoolByLoad();
- double ourLoad = sysStats.getAvarageUsagePerCPU();
- if (ourLoad > 1) {
- int cMax = ((ThreadPoolExecutor)executor).getMaximumPoolSize();
- if (cMax > 2) {
- ((ThreadPoolExecutor)executor).setMaximumPoolSize(Math.max(1, (int) ((double)cMax * 0.60D)));
- }
- } else {
- double sLoad = load / (double) PROC_COUNT;
- if (sLoad > 1.0D) {
- int cMax = ((ThreadPoolExecutor)executor).getMaximumPoolSize();
- if (cMax > 2) {
- ((ThreadPoolExecutor)executor).setMaximumPoolSize(Math.max(1, (int) ((double) cMax * 0.60D)));
- }
- } else if (sLoad < 0.9D && MAXIMUM_POOL_SIZE != ((ThreadPoolExecutor)executor).getMaximumPoolSize()) {
- ((ThreadPoolExecutor)executor).setMaximumPoolSize(MAXIMUM_POOL_SIZE);
- }
- log.info("external request, load:" + sLoad); //nocommit: remove when testing is done
-
- }
-
-
- ;
List<Future<Object>> results = executor.invokeAll(closeCalls, 8, TimeUnit.SECONDS);
for (Future<Object> future : results) {
@@ -578,6 +552,35 @@ public class ParWork implements Closeable {
}
}
+ public static void sizePoolByLoad() {
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) getExecutor();
+ double load = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
+ if (load < 0) {
+ log.warn("SystemLoadAverage not supported on this JVM");
+ load = 0;
+ }
+
+ double ourLoad = sysStats.getAvarageUsagePerCPU();
+ if (ourLoad > 1) {
+ int cMax = executor.getMaximumPoolSize();
+ if (cMax > 2) {
+ executor.setMaximumPoolSize(Math.max(1, (int) ((double)cMax * 0.60D)));
+ }
+ } else {
+ double sLoad = load / (double) PROC_COUNT;
+ if (sLoad > 1.0D) {
+ int cMax = executor.getMaximumPoolSize();
+ if (cMax > 2) {
+ executor.setMaximumPoolSize(Math.max(1, (int) ((double) cMax * 0.60D)));
+ }
+ } else if (sLoad < 0.9D && MAXIMUM_POOL_SIZE != executor.getMaximumPoolSize()) {
+ executor.setMaximumPoolSize(MAXIMUM_POOL_SIZE);
+ }
+ log.info("external request, load:" + sLoad); //nocommit: remove when testing is done
+
+ }
+ }
+
public static synchronized ExecutorService getExecutor() {
// if (executor != null) return executor;
ExecutorService exec = THREAD_LOCAL_EXECUTOR.get();
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java b/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
index 1d925f8..82b4a70 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java
@@ -80,6 +80,10 @@ public class SolrIgnoredThreadsFilter implements ThreadFilter {
if (threadName.startsWith("ScheduledTrigger")) {
return true;
}
+ if (threadName.contains("worker-SendThread")) {
+ return true;
+ }
+
return false;