svn commit: r1592415 - in /nutch/trunk: CHANGES.txt src/java/org/apache/nutch/fetcher/Fetcher.java

[sn...@apache.org]

Author: snagel
Date: Sun May  4 20:19:08 2014
New Revision: 1592415

URL: http://svn.apache.org/r1592415
Log:
NUTCH-1182 fetcher to log hung threads

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java

Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1592415&r1=1592414&r2=1592415&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sun May  4 20:19:08 2014
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Nutch Current Development
 
+* NUTCH-1182 fetcher to log hung threads (snagel)
+
 * NUTCH-1759 Upgrade to Crawler Commons 0.4 (jnioche)
 
 * NUTCH-1764 readdb to show command-line help if no action (-stats, -dump, etc.) given (Diaa via snagel)

Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1592415&r1=1592414&r2=1592415&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java Sun May  4 20:19:08 2014
@@ -133,6 +133,8 @@ public class Fetcher extends Configured 
   FetchItemQueues fetchQueues;
   QueueFeeder feeder;
 
+  LinkedList<FetcherThread> fetcherThreads = new LinkedList<FetcherThread>();
+
   /**
    * This class described the item to be fetched.
    */
@@ -1178,7 +1180,9 @@ public class Fetcher extends Configured 
     getConf().setBoolean(Protocol.CHECK_ROBOTS, false);
 
     for (int i = 0; i < threadCount; i++) {       // spawn threads
-      new FetcherThread(getConf()).start();
+      FetcherThread t = new FetcherThread(getConf());
+      fetcherThreads.add(t);
+      t.start();
     }
 
     // select a timeout that avoids a task timeout
@@ -1254,6 +1258,21 @@ public class Fetcher extends Configured 
       if ((System.currentTimeMillis() - lastRequestStart.get()) > timeout) {
         if (LOG.isWarnEnabled()) {
           LOG.warn("Aborting with "+activeThreads+" hung threads.");
+          for (int i = 0; i < fetcherThreads.size(); i++) {
+            FetcherThread thread = fetcherThreads.get(i);
+            if (thread.isAlive()) {
+              LOG.warn("Thread #" + i + " hung while processing " + thread.reprUrl);
+              if (LOG.isDebugEnabled()) {
+                StackTraceElement[] stack = thread.getStackTrace();
+                StringBuilder sb = new StringBuilder();
+                sb.append("Stack of thread #").append(i).append(":\n");
+                for (StackTraceElement s : stack) {
+                  sb.append(s.toString()).append('\n');
+                }
+                LOG.debug(sb.toString());
+              }
+            }
+          }
         }
         return;
       }