You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@tomcat.apache.org by ma...@apache.org on 2011/06/08 01:47:16 UTC
svn commit: r1133200 -
/tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java
Author: markt
Date: Tue Jun 7 23:47:16 2011
New Revision: 1133200
URL: http://svn.apache.org/viewvc?rev=1133200&view=rev
Log:
Enable for async requests.
Looking for stuck threads - no special async support required here.
When the asycn timeout is infinite requests may get stuck but that'll need a different detection mechanism.
Modified:
tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java
Modified: tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java
URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java?rev=1133200&r1=1133199&r2=1133200&view=diff
==============================================================================
--- tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java (original)
+++ tomcat/trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java Tue Jun 7 23:47:16 2011
@@ -1,302 +1,311 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.catalina.valves;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.servlet.ServletException;
-
-import org.apache.catalina.LifecycleException;
-import org.apache.catalina.connector.Request;
-import org.apache.catalina.connector.Response;
-import org.apache.juli.logging.Log;
-import org.apache.juli.logging.LogFactory;
-import org.apache.tomcat.util.res.StringManager;
-
-/**
- * This valve allows to detect requests that take a long time to process, which might
- * indicate that the thread that is processing it is stuck.
- * Based on code proposed by TomLu in Bugzilla entry #50306
- *
- * @author slaurent
- *
- */
-public class StuckThreadDetectionValve extends ValveBase {
-
- /**
- * The descriptive information related to this implementation.
- */
- private static final String info =
- "org.apache.catalina.valves.StuckThreadDetectionValve/1.0";
- /**
- * Logger
- */
- private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class);
-
- /**
- * The string manager for this package.
- */
- private static final StringManager sm =
- StringManager.getManager(Constants.Package);
-
- /**
- * Keeps count of the number of stuck threads detected
- */
- private final AtomicInteger stuckCount = new AtomicInteger(0);
-
- /**
- * In seconds. Default 600 (10 minutes).
- */
- private int threshold = 600;
-
- /**
- * The only references we keep to actual running Thread objects are in
- * this Map (which is automatically cleaned in invoke()s finally clause).
- * That way, Threads can be GC'ed, eventhough the Valve still thinks they
- * are stuck (caused by a long monitor interval)
- */
- private ConcurrentHashMap<Long, MonitoredThread> activeThreads =
- new ConcurrentHashMap<Long, MonitoredThread>();
- /**
- *
- */
- private Queue<CompletedStuckThread> completedStuckThreadsQueue =
- new ConcurrentLinkedQueue<CompletedStuckThread>();
-
- /**
- * Specify the threshold (in seconds) used when checking for stuck threads.
- * If <=0, the detection is disabled. The default is 600 seconds.
- *
- * @param threshold
- * The new threshold in seconds
- */
- public void setThreshold(int threshold) {
- this.threshold = threshold;
- }
-
- /**
- * @see #setThreshold(int)
- * @return The current threshold in seconds
- */
- public int getThreshold() {
- return threshold;
- }
-
- @Override
- protected void initInternal() throws LifecycleException {
- super.initInternal();
-
- if (log.isDebugEnabled()) {
- log.debug("Monitoring stuck threads with threshold = "
- + threshold
- + " sec");
- }
- }
-
- /**
- * Return descriptive information about this Valve implementation.
- */
- @Override
- public String getInfo() {
- return info;
- }
-
- private void notifyStuckThreadDetected(MonitoredThread monitoredThread,
- long activeTime, int numStuckThreads) {
- if (log.isWarnEnabled()) {
- String msg = sm.getString(
- "stuckThreadDetectionValve.notifyStuckThreadDetected",
- monitoredThread.getThread().getName(), activeTime,
- monitoredThread.getStartTime(), numStuckThreads,
- monitoredThread.getRequestUri(), threshold);
- // msg += "\n" + getStackTraceAsString(trace);
- Throwable th = new Throwable();
- th.setStackTrace(monitoredThread.getThread().getStackTrace());
- log.warn(msg, th);
- }
- }
-
- private void notifyStuckThreadCompleted(String threadName,
- long activeTime, int numStuckThreads) {
- if (log.isWarnEnabled()) {
- String msg = sm.getString(
- "stuckThreadDetectionValve.notifyStuckThreadCompleted",
- threadName, activeTime, numStuckThreads);
- // Since the "stuck thread notification" is warn, this should also
- // be warn
- log.warn(msg);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void invoke(Request request, Response response)
- throws IOException, ServletException {
-
- if (threshold <= 0) {
- // short-circuit if not monitoring stuck threads
- getNext().invoke(request, response);
- return;
- }
-
- // Save the thread/runnable
- // Keeping a reference to the thread object here does not prevent
- // GC'ing, as the reference is removed from the Map in the finally clause
-
- Long key = new Long(Thread.currentThread().getId());
- StringBuffer requestUrl = request.getRequestURL();
- if(request.getQueryString()!=null) {
- requestUrl.append("?");
- requestUrl.append(request.getQueryString());
- }
- MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(),
- requestUrl.toString());
- activeThreads.put(key, monitoredThread);
-
- try {
- getNext().invoke(request, response);
- } finally {
- activeThreads.remove(key);
- if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) {
- completedStuckThreadsQueue.add(
- new CompletedStuckThread(monitoredThread.getThread().getName(),
- monitoredThread.getActiveTimeInMillis()));
- }
- }
- }
-
- @Override
- public void backgroundProcess() {
- super.backgroundProcess();
-
- long thresholdInMillis = threshold * 1000;
-
- // Check monitored threads, being careful that the request might have
- // completed by the time we examine it
- for (MonitoredThread monitoredThread : activeThreads.values()) {
- long activeTime = monitoredThread.getActiveTimeInMillis();
-
- if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) {
- int numStuckThreads = stuckCount.incrementAndGet();
- notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads);
- }
- }
- // Check if any threads previously reported as stuck, have finished.
- for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll();
- completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) {
-
- int numStuckThreads = stuckCount.decrementAndGet();
- notifyStuckThreadCompleted(completedStuckThread.getName(),
- completedStuckThread.getTotalActiveTime(), numStuckThreads);
- }
- }
-
- public long[] getStuckThreadIds() {
- List<Long> idList = new ArrayList<Long>();
- for (MonitoredThread monitoredThread : activeThreads.values()) {
- if (monitoredThread.isMarkedAsStuck()) {
- idList.add(monitoredThread.getThread().getId());
- }
- }
-
- long[] result = new long[idList.size()];
- for (int i = 0; i < result.length; i++) {
- result[i] = idList.get(i);
- }
- return result;
- }
-
- private class MonitoredThread {
-
- /**
- * Reference to the thread to get a stack trace from background task
- */
- private final Thread thread;
- private final String requestUri;
- private final long start;
- private final AtomicInteger state = new AtomicInteger(
- MonitoredThreadState.RUNNING.ordinal());
-
- public MonitoredThread(Thread thread, String requestUri) {
- this.thread = thread;
- this.requestUri = requestUri;
- this.start = System.currentTimeMillis();
- }
-
- public Thread getThread() {
- return this.thread;
- }
-
- public String getRequestUri() {
- return requestUri;
- }
-
- public long getActiveTimeInMillis() {
- return System.currentTimeMillis() - start;
- }
-
- public Date getStartTime() {
- return new Date(start);
- }
-
- public boolean markAsStuckIfStillRunning() {
- return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(),
- MonitoredThreadState.STUCK.ordinal());
- }
-
- public MonitoredThreadState markAsDone() {
- int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal());
- return MonitoredThreadState.values()[val];
- }
-
- boolean isMarkedAsStuck() {
- return this.state.get() == MonitoredThreadState.STUCK.ordinal();
- }
- }
-
- private class CompletedStuckThread {
-
- private String threadName;
- private long totalActiveTime;
-
- public CompletedStuckThread(String threadName, long totalActiveTime) {
- this.threadName = threadName;
- this.totalActiveTime = totalActiveTime;
- }
-
- public String getName() {
- return this.threadName;
- }
-
- public long getTotalActiveTime() {
- return this.totalActiveTime;
- }
- }
-
- private enum MonitoredThreadState {
- RUNNING, STUCK, DONE;
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.catalina.valves;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.servlet.ServletException;
+
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.connector.Request;
+import org.apache.catalina.connector.Response;
+import org.apache.juli.logging.Log;
+import org.apache.juli.logging.LogFactory;
+import org.apache.tomcat.util.res.StringManager;
+
+/**
+ * This valve allows to detect requests that take a long time to process, which might
+ * indicate that the thread that is processing it is stuck.
+ * Based on code proposed by TomLu in Bugzilla entry #50306
+ *
+ * @author slaurent
+ *
+ */
+public class StuckThreadDetectionValve extends ValveBase {
+
+ /**
+ * The descriptive information related to this implementation.
+ */
+ private static final String info =
+ "org.apache.catalina.valves.StuckThreadDetectionValve/1.0";
+ /**
+ * Logger
+ */
+ private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class);
+
+ /**
+ * The string manager for this package.
+ */
+ private static final StringManager sm =
+ StringManager.getManager(Constants.Package);
+
+ /**
+ * Keeps count of the number of stuck threads detected
+ */
+ private final AtomicInteger stuckCount = new AtomicInteger(0);
+
+ /**
+ * In seconds. Default 600 (10 minutes).
+ */
+ private int threshold = 600;
+
+ /**
+ * The only references we keep to actual running Thread objects are in
+ * this Map (which is automatically cleaned in invoke()s finally clause).
+ * That way, Threads can be GC'ed, eventhough the Valve still thinks they
+ * are stuck (caused by a long monitor interval)
+ */
+ private ConcurrentHashMap<Long, MonitoredThread> activeThreads =
+ new ConcurrentHashMap<Long, MonitoredThread>();
+ /**
+ *
+ */
+ private Queue<CompletedStuckThread> completedStuckThreadsQueue =
+ new ConcurrentLinkedQueue<CompletedStuckThread>();
+
+ /**
+ * Specify the threshold (in seconds) used when checking for stuck threads.
+ * If <=0, the detection is disabled. The default is 600 seconds.
+ *
+ * @param threshold
+ * The new threshold in seconds
+ */
+ public void setThreshold(int threshold) {
+ this.threshold = threshold;
+ }
+
+ /**
+ * @see #setThreshold(int)
+ * @return The current threshold in seconds
+ */
+ public int getThreshold() {
+ return threshold;
+ }
+
+
+ /**
+ * Required to enable async support.
+ */
+ public StuckThreadDetectionValve() {
+ super(true);
+ }
+
+
+ @Override
+ protected void initInternal() throws LifecycleException {
+ super.initInternal();
+
+ if (log.isDebugEnabled()) {
+ log.debug("Monitoring stuck threads with threshold = "
+ + threshold
+ + " sec");
+ }
+ }
+
+ /**
+ * Return descriptive information about this Valve implementation.
+ */
+ @Override
+ public String getInfo() {
+ return info;
+ }
+
+ private void notifyStuckThreadDetected(MonitoredThread monitoredThread,
+ long activeTime, int numStuckThreads) {
+ if (log.isWarnEnabled()) {
+ String msg = sm.getString(
+ "stuckThreadDetectionValve.notifyStuckThreadDetected",
+ monitoredThread.getThread().getName(), activeTime,
+ monitoredThread.getStartTime(), numStuckThreads,
+ monitoredThread.getRequestUri(), threshold);
+ // msg += "\n" + getStackTraceAsString(trace);
+ Throwable th = new Throwable();
+ th.setStackTrace(monitoredThread.getThread().getStackTrace());
+ log.warn(msg, th);
+ }
+ }
+
+ private void notifyStuckThreadCompleted(String threadName,
+ long activeTime, int numStuckThreads) {
+ if (log.isWarnEnabled()) {
+ String msg = sm.getString(
+ "stuckThreadDetectionValve.notifyStuckThreadCompleted",
+ threadName, activeTime, numStuckThreads);
+ // Since the "stuck thread notification" is warn, this should also
+ // be warn
+ log.warn(msg);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void invoke(Request request, Response response)
+ throws IOException, ServletException {
+
+ if (threshold <= 0) {
+ // short-circuit if not monitoring stuck threads
+ getNext().invoke(request, response);
+ return;
+ }
+
+ // Save the thread/runnable
+ // Keeping a reference to the thread object here does not prevent
+ // GC'ing, as the reference is removed from the Map in the finally clause
+
+ Long key = new Long(Thread.currentThread().getId());
+ StringBuffer requestUrl = request.getRequestURL();
+ if(request.getQueryString()!=null) {
+ requestUrl.append("?");
+ requestUrl.append(request.getQueryString());
+ }
+ MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(),
+ requestUrl.toString());
+ activeThreads.put(key, monitoredThread);
+
+ try {
+ getNext().invoke(request, response);
+ } finally {
+ activeThreads.remove(key);
+ if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) {
+ completedStuckThreadsQueue.add(
+ new CompletedStuckThread(monitoredThread.getThread().getName(),
+ monitoredThread.getActiveTimeInMillis()));
+ }
+ }
+ }
+
+ @Override
+ public void backgroundProcess() {
+ super.backgroundProcess();
+
+ long thresholdInMillis = threshold * 1000;
+
+ // Check monitored threads, being careful that the request might have
+ // completed by the time we examine it
+ for (MonitoredThread monitoredThread : activeThreads.values()) {
+ long activeTime = monitoredThread.getActiveTimeInMillis();
+
+ if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) {
+ int numStuckThreads = stuckCount.incrementAndGet();
+ notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads);
+ }
+ }
+ // Check if any threads previously reported as stuck, have finished.
+ for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll();
+ completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) {
+
+ int numStuckThreads = stuckCount.decrementAndGet();
+ notifyStuckThreadCompleted(completedStuckThread.getName(),
+ completedStuckThread.getTotalActiveTime(), numStuckThreads);
+ }
+ }
+
+ public long[] getStuckThreadIds() {
+ List<Long> idList = new ArrayList<Long>();
+ for (MonitoredThread monitoredThread : activeThreads.values()) {
+ if (monitoredThread.isMarkedAsStuck()) {
+ idList.add(monitoredThread.getThread().getId());
+ }
+ }
+
+ long[] result = new long[idList.size()];
+ for (int i = 0; i < result.length; i++) {
+ result[i] = idList.get(i);
+ }
+ return result;
+ }
+
+ private class MonitoredThread {
+
+ /**
+ * Reference to the thread to get a stack trace from background task
+ */
+ private final Thread thread;
+ private final String requestUri;
+ private final long start;
+ private final AtomicInteger state = new AtomicInteger(
+ MonitoredThreadState.RUNNING.ordinal());
+
+ public MonitoredThread(Thread thread, String requestUri) {
+ this.thread = thread;
+ this.requestUri = requestUri;
+ this.start = System.currentTimeMillis();
+ }
+
+ public Thread getThread() {
+ return this.thread;
+ }
+
+ public String getRequestUri() {
+ return requestUri;
+ }
+
+ public long getActiveTimeInMillis() {
+ return System.currentTimeMillis() - start;
+ }
+
+ public Date getStartTime() {
+ return new Date(start);
+ }
+
+ public boolean markAsStuckIfStillRunning() {
+ return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(),
+ MonitoredThreadState.STUCK.ordinal());
+ }
+
+ public MonitoredThreadState markAsDone() {
+ int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal());
+ return MonitoredThreadState.values()[val];
+ }
+
+ boolean isMarkedAsStuck() {
+ return this.state.get() == MonitoredThreadState.STUCK.ordinal();
+ }
+ }
+
+ private class CompletedStuckThread {
+
+ private String threadName;
+ private long totalActiveTime;
+
+ public CompletedStuckThread(String threadName, long totalActiveTime) {
+ this.threadName = threadName;
+ this.totalActiveTime = totalActiveTime;
+ }
+
+ public String getName() {
+ return this.threadName;
+ }
+
+ public long getTotalActiveTime() {
+ return this.totalActiveTime;
+ }
+ }
+
+ private enum MonitoredThreadState {
+ RUNNING, STUCK, DONE;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org