You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2020/07/09 19:13:15 UTC
[lucene-solr] branch master updated: SOLR-14635: ThreadDumpHandler
has been enhanced to show lock ownership
This is an automated email from the ASF dual-hosted git repository.
hossman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 5c6314a SOLR-14635: ThreadDumpHandler has been enhanced to show lock ownership
5c6314a is described below
commit 5c6314a970f9a6a07aee5a14851f3b0f9fbe02fb
Author: Chris Hostetter <ho...@apache.org>
AuthorDate: Thu Jul 9 12:13:02 2020 -0700
SOLR-14635: ThreadDumpHandler has been enhanced to show lock ownership
---
solr/CHANGES.txt | 1 +
.../solr/handler/admin/ThreadDumpHandler.java | 54 ++++++--
.../solr/handler/admin/ThreadDumpHandlerTest.java | 152 +++++++++++++++++++++
3 files changed, 198 insertions(+), 9 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4dcc538..68880e9 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -119,6 +119,7 @@ Improvements
logs from the receiving coordinator node with those from downstream shard requests. This can be disabled by providing a
disableRequestId=true request parameter. (Jason Gerlowski)
+* SOLR-14635: ThreadDumpHandler has been enhanced to show lock ownership (hossman)
Optimizations
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java
index bb5b3ee..e13a0a0 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java
@@ -19,7 +19,10 @@ package org.apache.solr.handler.admin;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
+import java.lang.management.LockInfo;
import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Locale;
import org.apache.solr.common.util.NamedList;
@@ -54,7 +57,7 @@ public class ThreadDumpHandler extends RequestHandlerBase
// Deadlocks
ThreadInfo[] tinfos;
- long[] tids = tmbean.findMonitorDeadlockedThreads();
+ long[] tids = tmbean.findDeadlockedThreads();
if (tids != null) {
tinfos = tmbean.getThreadInfo(tids, Integer.MAX_VALUE);
NamedList<SimpleOrderedMap<Object>> lst = new NamedList<>();
@@ -67,8 +70,8 @@ public class ThreadDumpHandler extends RequestHandlerBase
}
// Now show all the threads....
- tids = tmbean.getAllThreadIds();
- tinfos = tmbean.getThreadInfo(tids, Integer.MAX_VALUE);
+
+ tinfos = tmbean.dumpAllThreads(true, true);
NamedList<SimpleOrderedMap<Object>> lst = new NamedList<>();
for (ThreadInfo ti : tinfos) {
if (ti != null) {
@@ -91,8 +94,47 @@ public class ThreadDumpHandler extends RequestHandlerBase
info.add( "state", ti.getThreadState().toString() );
if (ti.getLockName() != null) {
+ // TODO: this is redundent with lock-waiting below .. deprecate & remove
+ // TODO: (but first needs UI change)
info.add( "lock", ti.getLockName() );
}
+ { final LockInfo lockInfo = ti.getLockInfo();
+ if (null != lockInfo) {
+ final SimpleOrderedMap<Object> lock = new SimpleOrderedMap<>();
+ info.add("lock-waiting", lock);
+ lock.add(NAME, lockInfo.toString());
+ if (-1 == ti.getLockOwnerId() && null == ti.getLockOwnerName()) {
+ lock.add("owner", null );
+ } else {
+ final SimpleOrderedMap<Object> owner = new SimpleOrderedMap<>();
+ lock.add("owner", owner);
+ owner.add(NAME, ti.getLockOwnerName());
+ owner.add( ID, ti.getLockOwnerId() );
+ }
+ }
+ }
+ { final LockInfo[] synchronizers = ti.getLockedSynchronizers();
+ if (0 < synchronizers.length) {
+ final List<String> locks = new ArrayList<>(synchronizers.length);
+ info.add("synchronizers-locked", locks);
+ for (LockInfo sync : synchronizers) {
+ locks.add(sync.toString());
+ }
+ }
+ }
+ { final LockInfo[] monitors = ti.getLockedMonitors();
+ if (0 < monitors.length) {
+ final List<String> locks = new ArrayList<>(monitors.length);
+ info.add("monitors-locked", locks);
+ for (LockInfo monitor : monitors) {
+ locks.add(monitor.toString());
+ }
+ }
+ }
+
+
+
+
if (ti.isSuspended()) {
info.add( "suspended", true );
}
@@ -105,12 +147,6 @@ public class ThreadDumpHandler extends RequestHandlerBase
info.add( "userTime", formatNanos(tmbean.getThreadUserTime(tid)) );
}
- if (ti.getLockOwnerName() != null) {
- SimpleOrderedMap<Object> owner = new SimpleOrderedMap<>();
- owner.add(NAME, ti.getLockOwnerName());
- owner.add( ID, ti.getLockOwnerId() );
- }
-
// Add the stack trace
int i=0;
String[] trace = new String[ti.getStackTrace().length];
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ThreadDumpHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ThreadDumpHandlerTest.java
new file mode 100644
index 0000000..ef28b8c
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/ThreadDumpHandlerTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.BeforeClass;
+
+public class ThreadDumpHandlerTest extends SolrTestCaseJ4 {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig.xml", "schema.xml");
+ }
+
+
+ public void testMonitor() throws Exception {
+ assumeTrue("monitor checking not supported on this JVM",
+ ManagementFactory.getThreadMXBean().isObjectMonitorUsageSupported());
+
+ /** unique class name to show up as a lock class name in output */
+ final class TestMonitorStruct { /* empty */ }
+
+ final List<String> failures = new ArrayList<>();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final Object monitor = new TestMonitorStruct();
+ final Thread owner = new Thread(() -> {
+ synchronized (monitor) {
+ log.info("monitor owner waiting for latch to release me...");
+ try {
+ if ( ! latch.await(5, TimeUnit.SECONDS ) ){
+ failures.add("owner: never saw latch release");
+ }
+ } catch (InterruptedException ie) {
+ failures.add("owner: " + ie.toString());
+ }
+ }
+ }, "test-thread-monitor-owner");
+ final Thread blocked = new Thread(() -> {
+ log.info("blocked waiting for monitor...");
+ synchronized (monitor) {
+ log.info("monitor now unblocked");
+ }
+ }, "test-thread-monitor-blocked");
+ try {
+ owner.start();
+ blocked.start();
+
+ assertQ(req("qt", "/admin/threads", "indent", "true")
+ // monitor owner thread (which is also currently waiting on CountDownLatch)
+ , "//lst[@name='thread'][str[@name='name'][.='test-thread-monitor-owner']]"
+ + " [lst[@name='lock-waiting'][null[@name='owner']]]" // latch
+ + " [arr[@name='monitors-locked']/str[contains(.,'TestMonitorStruct')]]"
+ // blocked thread, waiting on the monitor
+ , "//lst[@name='thread'][str[@name='name'][.='test-thread-monitor-blocked']]"
+ + " [lst[@name='lock-waiting'][lst[@name='owner']/str[.='test-thread-monitor-owner']]]"
+ );
+
+ } finally {
+ latch.countDown();
+ owner.join(1000);
+ assertFalse("owner is still alive", owner.isAlive());
+ blocked.join(1000);
+ assertFalse("blocked is still alive", blocked.isAlive());
+ }
+ }
+
+
+ public void testOwnableSync() throws Exception {
+ assumeTrue("ownable sync checking not supported on this JVM",
+ ManagementFactory.getThreadMXBean().isSynchronizerUsageSupported());
+
+ /** unique class name to show up as a lock class name in output */
+ final class TestReentrantLockStruct extends ReentrantLock { /* empty */ }
+
+ final List<String> failures = new ArrayList<>();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final ReentrantLock lock = new ReentrantLock();
+ final Thread owner = new Thread(() -> {
+ lock.lock();
+ try {
+ log.info("lock owner waiting for latch to release me...");
+ try {
+ if ( ! latch.await(5, TimeUnit.SECONDS ) ){
+ failures.add("owner: never saw latch release");
+ }
+ } catch (InterruptedException ie) {
+ failures.add("owner: " + ie.toString());
+ }
+ } finally {
+ lock.unlock();
+ }
+ }, "test-thread-sync-lock-owner");
+ final Thread blocked = new Thread(() -> {
+ log.info("blocked waiting for lock...");
+ lock.lock();
+ try {
+ log.info("lock now unblocked");
+ } finally {
+ lock.unlock();
+ }
+ }, "test-thread-sync-lock-blocked");
+ try {
+ owner.start();
+ blocked.start();
+
+ assertQ(req("qt", "/admin/threads", "indent", "true")
+ // lock owner thread (which is also currently waiting on CountDownLatch)
+ , "//lst[@name='thread'][str[@name='name'][.='test-thread-sync-lock-owner']]"
+ + " [lst[@name='lock-waiting'][null[@name='owner']]]" // latch
+ + " [arr[@name='synchronizers-locked']/str[contains(.,'ReentrantLock')]]"
+ // blocked thread, waiting on the lock
+ , "//lst[@name='thread'][str[@name='name'][.='test-thread-sync-lock-blocked']]"
+ + " [lst[@name='lock-waiting'][lst[@name='owner']/str[.='test-thread-sync-lock-owner']]]"
+ );
+
+ } finally {
+ latch.countDown();
+ owner.join(1000);
+ assertFalse("owner is still alive", owner.isAlive());
+ blocked.join(1000);
+ assertFalse("blocked is still alive", blocked.isAlive());
+ }
+ }
+
+}