You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by ol...@apache.org on 2008/11/04 16:11:42 UTC
svn commit: r711289 - in /incubator/droids/trunk/src: java/
java/org/apache/droids/ java/org/apache/droids/impl/
java/org/apache/droids/protocol/http/ java/org/apache/droids/robot/crawler/
java/org/apache/droids/robot/walker/ test/java/org/apache/droid...
Author: olegk
Date: Tue Nov 4 08:11:41 2008
New Revision: 711289
URL: http://svn.apache.org/viewvc?rev=711289&view=rev
Log:
All console output to go through the commons logging API
Modified:
incubator/droids/trunk/src/java/log4j.properties
incubator/droids/trunk/src/java/org/apache/droids/AbstractDroid.java
incubator/droids/trunk/src/java/org/apache/droids/SimpleThreads.java
incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
incubator/droids/trunk/src/java/org/apache/droids/protocol/http/Http.java
incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java
incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java
incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java
incubator/droids/trunk/src/test/java/org/apache/droids/localserver/LocalHttpServer.java
Modified: incubator/droids/trunk/src/java/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/log4j.properties?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/log4j.properties (original)
+++ incubator/droids/trunk/src/java/log4j.properties Tue Nov 4 08:11:41 2008
@@ -17,7 +17,7 @@
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - <%m>%n
+log4j.appender.stdout.layout.ConversionPattern=%d %p [%t] - <%m>%n
log4j.appender.logfile=org.apache.log4j.RollingFileAppender
log4j.appender.logfile.File=log.txt
@@ -26,4 +26,4 @@
log4j.appender.logfile.MaxBackupIndex=3
# Pattern to output: date priority [category] - message
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
-log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
\ No newline at end of file
+log4j.appender.logfile.layout.ConversionPattern=%d %p [%t] - %m%n
\ No newline at end of file
Modified: incubator/droids/trunk/src/java/org/apache/droids/AbstractDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/AbstractDroid.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/AbstractDroid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/AbstractDroid.java Tue Nov 4 08:11:41 2008
@@ -16,12 +16,16 @@
*/
package org.apache.droids;
-import org.apache.droids.api.*;
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Task;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.helper.Loggable;
/**
* Manage common tasks in standard Droids
*/
-public abstract class AbstractDroid<T extends Task> implements Droid<T>
+public abstract class AbstractDroid<T extends Task> extends Loggable implements Droid<T>
{
protected final TaskQueue<T> queue;
protected final TaskMaster<T> taskMaster;
@@ -37,11 +41,6 @@
taskMaster.processAllTasks(queue, this);
}
- public void finished()
- {
- System.out.println( "FINISHED!!!" );
- }
-
public final TaskQueue<T> getQueue() {
return queue;
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/SimpleThreads.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/SimpleThreads.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/SimpleThreads.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/SimpleThreads.java Tue Nov 4 08:11:41 2008
@@ -3,7 +3,7 @@
public class SimpleThreads {
//Display a message, preceded by the name of the current thread
- public static void threadMessage(String message) {
+ private static void threadMessage(String message) {
String threadName = Thread.currentThread().getName();
System.out.format("%s: %s%n", threadName, message);
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java Tue Nov 4 08:11:41 2008
@@ -22,10 +22,16 @@
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.droids.SimpleThreads;
-import org.apache.droids.api.*;
+import org.apache.droids.api.DelayTimer;
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Task;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
+import org.apache.droids.helper.Loggable;
-public class MultiThreadedTaskMaster<T extends Task> implements TaskMaster<T>
+public class MultiThreadedTaskMaster<T extends Task>
+ extends Loggable implements TaskMaster<T>
{
private static final long KEEP_ALIVE = 50000L;
@@ -57,7 +63,7 @@
this.runningWorker = new ConcurrentHashMap<Long, WorkerRunner>();
startWorkers();
- SimpleThreads.threadMessage("Finshed invocation, waiting for workers to finish.");
+ log.info("Finshed invocation, waiting for workers to finish.");
}
@@ -71,12 +77,16 @@
//int y = worker.getDepth() + 1;
pool.remove(worker);
runningWorker.remove(id);
- SimpleThreads.threadMessage("Worker \"" + id + "\" has finished.");
+ if (log.isInfoEnabled()) {
+ log.info("Worker \"" + id + "\" has finished.");
+ }
if (runningWorker.size() == 0 & !queue.hasNext()) {
shutdownAndAwaitTermination();
long elapsed = System.currentTimeMillis() - startedWorking.getTime();
- SimpleThreads.threadMessage("All threads have finished. (elapsed:"+elapsed+")" );
+ if (log.isInfoEnabled()) {
+ log.info("All threads have finished. (elapsed:" + elapsed + ")" );
+ }
finishedWorking = new Date();
droid.finished();
} else if (queue.hasNext()) {
@@ -110,10 +120,12 @@
WorkerRunner runner = new WorkerRunner();
runningWorker.put(runner.getId(), runner);
pool.execute(runner);
- SimpleThreads.threadMessage("starting "+runner.getId());
+ if (log.isInfoEnabled()) {
+ log.info("starting " + runner.getId());
+ }
}
- catch (Exception e) {
- SimpleThreads.threadMessage(e.getMessage());
+ catch (Exception ex) {
+ log.error(ex.getMessage(), ex);
}
}
}
@@ -150,16 +162,16 @@
* threads.
*/
protected void shutdownAndAwaitTermination() {
- SimpleThreads.threadMessage("SHUTTING DOWN");
+ log.info("SHUTTING DOWN");
pool.shutdown(); // Disable new tasks from being submitted
try {
// Wait a while for existing tasks to terminate
if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
- SimpleThreads.threadMessage("SHUT DOWN NOW");
+ log.info("SHUT DOWN NOW");
pool.shutdownNow(); // Cancel currently executing tasks
// Wait a while for tasks to respond to being canceled
if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
- SimpleThreads.threadMessage("Pool did not terminate");
+ log.info("Pool did not terminate");
}
}
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/protocol/http/Http.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/protocol/http/Http.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/protocol/http/Http.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/protocol/http/Http.java Tue Nov 4 08:11:41 2008
@@ -22,7 +22,6 @@
import java.net.MalformedURLException;
import java.net.URL;
-import org.apache.droids.SimpleThreads;
import org.apache.droids.api.Protocol;
import org.apache.droids.net.UrlHelper;
import org.apache.droids.protocol.HttpBase;
@@ -91,7 +90,9 @@
}
}
String message = (test) ? "allowed" : "denied";
- SimpleThreads.threadMessage("Url is " + message);
+ if (log.isInfoEnabled()) {
+ log.info("Url is " + message);
+ }
return test;
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java Tue Nov 4 08:11:41 2008
@@ -69,7 +69,7 @@
public void finished()
{
- System.out.println( "FINISHED!!!" );
+ log.info( "FINISHED!!!" );
}
public CrawlingWorker getNewWorker() {
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java Tue Nov 4 08:11:41 2008
@@ -24,11 +24,15 @@
import java.util.HashMap;
import java.util.Map;
-import org.apache.droids.SimpleThreads;
-import org.apache.droids.api.*;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.Parse;
+import org.apache.droids.api.Parser;
+import org.apache.droids.api.Protocol;
+import org.apache.droids.api.Worker;
+import org.apache.droids.helper.Loggable;
import org.apache.droids.helper.factories.URLFiltersFactory;
-public class CrawlingWorker implements Worker<Link>
+public class CrawlingWorker extends Loggable implements Worker<Link>
{
private final CrawlingDroid droid;
@@ -41,21 +45,27 @@
{
InputStream openStream = null;
final String userAgent = this.getClass().getCanonicalName();
- SimpleThreads.threadMessage("Starting " + userAgent);
+ if (log.isInfoEnabled()) {
+ log.info("Starting " + userAgent);
+ }
try {
String url = link.getId();
final Protocol protocol = droid.getProtocolFactory().getProtocol(url);
openStream = protocol.openStream(url);
if (protocol.isAllowed(url)) {
String contentType = protocol.getContentType(url);
- SimpleThreads.threadMessage("contentType " + contentType);
+ if (log.isInfoEnabled()) {
+ log.info("contentType " + contentType);
+ }
if (contentType==null){
- SimpleThreads.threadMessage("missing contentType... can't parse..." );
+ log.info("missing contentType... can't parse...");
}
else {
Parser parser = droid.getParserFactory().getParser(contentType);
if( parser == null ) {
- SimpleThreads.threadMessage("could not find parser for: "+contentType);
+ if (log.isInfoEnabled()) {
+ log.info("could not find parser for: " + contentType);
+ }
}
else {
Parse parse = parser.getParse(openStream, link);
@@ -68,7 +78,7 @@
}
}
else {
- SimpleThreads.threadMessage("stopping processing since"
+ log.info("Stopping processing since"
+ " bots are not allowed for this url.");
}
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java Tue Nov 4 08:11:41 2008
@@ -18,12 +18,12 @@
import java.io.File;
-import org.apache.droids.SimpleThreads;
import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.exception.InvalidTaskException;
+import org.apache.droids.helper.Loggable;
-public class FileWorker implements Worker<FileTask>
+public class FileWorker extends Loggable implements Worker<FileTask>
{
final TaskQueue<FileTask> queue;
@@ -47,7 +47,9 @@
}
}
else {
- SimpleThreads.threadMessage( "FILE: "+file.getAbsolutePath() );
+ if (log.isInfoEnabled()) {
+ log.info( "FILE: "+file.getAbsolutePath() );
+ }
}
}
}
\ No newline at end of file
Modified: incubator/droids/trunk/src/test/java/org/apache/droids/localserver/LocalHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/test/java/org/apache/droids/localserver/LocalHttpServer.java?rev=711289&r1=711288&r2=711289&view=diff
==============================================================================
--- incubator/droids/trunk/src/test/java/org/apache/droids/localserver/LocalHttpServer.java (original)
+++ incubator/droids/trunk/src/test/java/org/apache/droids/localserver/LocalHttpServer.java Tue Nov 4 08:11:41 2008
@@ -26,6 +26,7 @@
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.droids.helper.Loggable;
import org.apache.http.ConnectionReuseStrategy;
import org.apache.http.HttpException;
import org.apache.http.HttpServerConnection;
@@ -50,7 +51,7 @@
/**
* Local HTTP server for tests that require one.
*/
-public class LocalHttpServer
+public class LocalHttpServer extends Loggable
{
/**
@@ -162,9 +163,8 @@
try {
this.servicedSocket.close();
}
- catch (IOException iox) {
- System.out.println("error stopping " + this);
- iox.printStackTrace(System.out);
+ catch (IOException ex) {
+ log.error(ex.getMessage(), ex);
}
finally {
this.servicedSocket = null;
@@ -237,11 +237,10 @@
try {
accept();
}
- catch (Exception e) {
+ catch (Exception ex) {
ServerSocket ssock = servicedSocket;
if ((ssock != null) && !ssock.isClosed()) {
- System.out.println(LocalHttpServer.this.toString() + " could not accept");
- e.printStackTrace(System.out);
+ log.error(LocalHttpServer.this.toString() + " could not accept", ex);
}
// otherwise ignore the exception silently
break;
Re: Logging wars; was Re: svn commit: r711289
Posted by Ryan McKinley <ry...@gmail.com>.
On Nov 4, 2008, at 12:05 PM, Oleg Kalnichevski wrote:
> On Tue, 2008-11-04 at 11:16 -0500, Ryan McKinley wrote:
>> I had hoped we could use SLF4J:
>> https://issues.apache.org/jira/browse/DROIDS-9
>>
>> but since commons http component already depends on commons logging,
>> perhaps it is best to stick with that...
>>
>>
>
> Ryan
>
> CL and SLF4J are not mutually exclusive. As far as I know SLF4J
> provides
> a CL bridge which is (expected to be) fully binary compatible with the
> official CL implementation. One can compile against CL and deploy with
> SLF4J. This is one of the reasons why we decided to stick to CL for
> HttpClient 4.0.
>
yes, they have:
http://www.slf4j.org/legacy.html
you can install 'jcl-over-slf4j.jar' to use that
since HttpClient uses CL, we are stuck with CL in the classpath
anyway, so i guess there is not much to worry about ;)
Commons logging it is...
Logging wars; was Re: svn commit: r711289
Posted by Oleg Kalnichevski <ol...@apache.org>.
On Tue, 2008-11-04 at 11:16 -0500, Ryan McKinley wrote:
> I had hoped we could use SLF4J:
> https://issues.apache.org/jira/browse/DROIDS-9
>
> but since commons http component already depends on commons logging,
> perhaps it is best to stick with that...
>
>
Ryan
CL and SLF4J are not mutually exclusive. As far as I know SLF4J provides
a CL bridge which is (expected to be) fully binary compatible with the
official CL implementation. One can compile against CL and deploy with
SLF4J. This is one of the reasons why we decided to stick to CL for
HttpClient 4.0.
Oleg
Re: svn commit: r711289 - in /incubator/droids/trunk/src: java/ java/org/apache/droids/ java/org/apache/droids/impl/ java/org/apache/droids/protocol/http/ java/org/apache/droids/robot/crawler/ java/org/apache/droids/robot/walker/ test/java/org/apache/droid...
Posted by Ryan McKinley <ry...@gmail.com>.
I had hoped we could use SLF4J:
https://issues.apache.org/jira/browse/DROIDS-9
but since commons http component already depends on commons logging,
perhaps it is best to stick with that...
On Nov 4, 2008, at 11:11 AM, olegk@apache.org wrote:
> Author: olegk
> Date: Tue Nov 4 08:11:41 2008
> New Revision: 711289
>
> URL: http://svn.apache.org/viewvc?rev=711289&view=rev
> Log:
> All console output to go through the commons logging API
>
> Modified:
> incubator/droids/trunk/src/java/log4j.properties
> incubator/droids/trunk/src/java/org/apache/droids/
> AbstractDroid.java
> incubator/droids/trunk/src/java/org/apache/droids/
> SimpleThreads.java
> incubator/droids/trunk/src/java/org/apache/droids/impl/
> MultiThreadedTaskMaster.java
> incubator/droids/trunk/src/java/org/apache/droids/protocol/http/
> Http.java
> incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/
> CrawlingDroid.java
> incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/
> CrawlingWorker.java
> incubator/droids/trunk/src/java/org/apache/droids/robot/walker/
> FileWorker.java
> incubator/droids/trunk/src/test/java/org/apache/droids/
> localserver/LocalHttpServer.java
>
> Modified: incubator/droids/trunk/src/java/log4j.properties
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/log4j.properties?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/log4j.properties (original)
> +++ incubator/droids/trunk/src/java/log4j.properties Tue Nov 4
> 08:11:41 2008
> @@ -17,7 +17,7 @@
>
> log4j.appender.stdout=org.apache.log4j.ConsoleAppender
> log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
> -log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - <%m>%n
> +log4j.appender.stdout.layout.ConversionPattern=%d %p [%t] - <%m>%n
>
> log4j.appender.logfile=org.apache.log4j.RollingFileAppender
> log4j.appender.logfile.File=log.txt
> @@ -26,4 +26,4 @@
> log4j.appender.logfile.MaxBackupIndex=3
> # Pattern to output: date priority [category] - message
> log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
> -log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
> \ No newline at end of file
> +log4j.appender.logfile.layout.ConversionPattern=%d %p [%t] - %m%n
> \ No newline at end of file
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/
> AbstractDroid.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/AbstractDroid.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/
> AbstractDroid.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/
> AbstractDroid.java Tue Nov 4 08:11:41 2008
> @@ -16,12 +16,16 @@
> */
> package org.apache.droids;
>
> -import org.apache.droids.api.*;
> +import org.apache.droids.api.Droid;
> +import org.apache.droids.api.Task;
> +import org.apache.droids.api.TaskMaster;
> +import org.apache.droids.api.TaskQueue;
> +import org.apache.droids.helper.Loggable;
>
> /**
> * Manage common tasks in standard Droids
> */
> -public abstract class AbstractDroid<T extends Task> implements
> Droid<T>
> +public abstract class AbstractDroid<T extends Task> extends
> Loggable implements Droid<T>
> {
> protected final TaskQueue<T> queue;
> protected final TaskMaster<T> taskMaster;
> @@ -37,11 +41,6 @@
> taskMaster.processAllTasks(queue, this);
> }
>
> - public void finished()
> - {
> - System.out.println( "FINISHED!!!" );
> - }
> -
> public final TaskQueue<T> getQueue() {
> return queue;
> }
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/
> SimpleThreads.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/SimpleThreads.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/
> SimpleThreads.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/
> SimpleThreads.java Tue Nov 4 08:11:41 2008
> @@ -3,7 +3,7 @@
> public class SimpleThreads {
>
> //Display a message, preceded by the name of the current thread
> - public static void threadMessage(String message) {
> + private static void threadMessage(String message) {
> String threadName = Thread.currentThread().getName();
> System.out.format("%s: %s%n", threadName, message);
> }
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/impl/
> MultiThreadedTaskMaster.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/impl/
> MultiThreadedTaskMaster.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/impl/
> MultiThreadedTaskMaster.java Tue Nov 4 08:11:41 2008
> @@ -22,10 +22,16 @@
> import java.util.concurrent.ThreadPoolExecutor;
> import java.util.concurrent.TimeUnit;
>
> -import org.apache.droids.SimpleThreads;
> -import org.apache.droids.api.*;
> +import org.apache.droids.api.DelayTimer;
> +import org.apache.droids.api.Droid;
> +import org.apache.droids.api.Task;
> +import org.apache.droids.api.TaskMaster;
> +import org.apache.droids.api.TaskQueue;
> +import org.apache.droids.api.Worker;
> +import org.apache.droids.helper.Loggable;
>
> -public class MultiThreadedTaskMaster<T extends Task> implements
> TaskMaster<T>
> +public class MultiThreadedTaskMaster<T extends Task>
> + extends Loggable implements
> TaskMaster<T>
> {
> private static final long KEEP_ALIVE = 50000L;
>
> @@ -57,7 +63,7 @@
> this.runningWorker = new ConcurrentHashMap<Long, WorkerRunner>();
>
> startWorkers();
> - SimpleThreads.threadMessage("Finshed invocation, waiting for
> workers to finish.");
> + log.info("Finshed invocation, waiting for workers to finish.");
> }
>
>
> @@ -71,12 +77,16 @@
> //int y = worker.getDepth() + 1;
> pool.remove(worker);
> runningWorker.remove(id);
> - SimpleThreads.threadMessage("Worker \"" + id + "\" has
> finished.");
> + if (log.isInfoEnabled()) {
> + log.info("Worker \"" + id + "\" has finished.");
> + }
> if (runningWorker.size() == 0 & !queue.hasNext()) {
> shutdownAndAwaitTermination();
>
> long elapsed = System.currentTimeMillis() -
> startedWorking.getTime();
> - SimpleThreads.threadMessage("All threads have finished.
> (elapsed:"+elapsed+")" );
> + if (log.isInfoEnabled()) {
> + log.info("All threads have finished. (elapsed:" +
> elapsed + ")" );
> + }
> finishedWorking = new Date();
> droid.finished();
> } else if (queue.hasNext()) {
> @@ -110,10 +120,12 @@
> WorkerRunner runner = new WorkerRunner();
> runningWorker.put(runner.getId(), runner);
> pool.execute(runner);
> - SimpleThreads.threadMessage("starting "+runner.getId());
> + if (log.isInfoEnabled()) {
> + log.info("starting " + runner.getId());
> + }
> }
> - catch (Exception e) {
> - SimpleThreads.threadMessage(e.getMessage());
> + catch (Exception ex) {
> + log.error(ex.getMessage(), ex);
> }
> }
> }
> @@ -150,16 +162,16 @@
> * threads.
> */
> protected void shutdownAndAwaitTermination() {
> - SimpleThreads.threadMessage("SHUTTING DOWN");
> + log.info("SHUTTING DOWN");
> pool.shutdown(); // Disable new tasks from being submitted
> try {
> // Wait a while for existing tasks to terminate
> if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
> - SimpleThreads.threadMessage("SHUT DOWN NOW");
> + log.info("SHUT DOWN NOW");
> pool.shutdownNow(); // Cancel currently executing tasks
> // Wait a while for tasks to respond to being canceled
> if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
> - SimpleThreads.threadMessage("Pool did not terminate");
> + log.info("Pool did not terminate");
> }
> }
> }
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/protocol/
> http/Http.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/protocol/http/Http.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/protocol/http/
> Http.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/protocol/http/
> Http.java Tue Nov 4 08:11:41 2008
> @@ -22,7 +22,6 @@
> import java.net.MalformedURLException;
> import java.net.URL;
>
> -import org.apache.droids.SimpleThreads;
> import org.apache.droids.api.Protocol;
> import org.apache.droids.net.UrlHelper;
> import org.apache.droids.protocol.HttpBase;
> @@ -91,7 +90,9 @@
> }
> }
> String message = (test) ? "allowed" : "denied";
> - SimpleThreads.threadMessage("Url is " + message);
> + if (log.isInfoEnabled()) {
> + log.info("Url is " + message);
> + }
> return test;
> }
>
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/
> crawler/CrawlingDroid.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/
> CrawlingDroid.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/
> CrawlingDroid.java Tue Nov 4 08:11:41 2008
> @@ -69,7 +69,7 @@
>
> public void finished()
> {
> - System.out.println( "FINISHED!!!" );
> + log.info( "FINISHED!!!" );
> }
>
> public CrawlingWorker getNewWorker() {
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/
> crawler/CrawlingWorker.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/
> CrawlingWorker.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/
> CrawlingWorker.java Tue Nov 4 08:11:41 2008
> @@ -24,11 +24,15 @@
> import java.util.HashMap;
> import java.util.Map;
>
> -import org.apache.droids.SimpleThreads;
> -import org.apache.droids.api.*;
> +import org.apache.droids.api.Link;
> +import org.apache.droids.api.Parse;
> +import org.apache.droids.api.Parser;
> +import org.apache.droids.api.Protocol;
> +import org.apache.droids.api.Worker;
> +import org.apache.droids.helper.Loggable;
> import org.apache.droids.helper.factories.URLFiltersFactory;
>
> -public class CrawlingWorker implements Worker<Link>
> +public class CrawlingWorker extends Loggable implements Worker<Link>
> {
> private final CrawlingDroid droid;
>
> @@ -41,21 +45,27 @@
> {
> InputStream openStream = null;
> final String userAgent = this.getClass().getCanonicalName();
> - SimpleThreads.threadMessage("Starting " + userAgent);
> + if (log.isInfoEnabled()) {
> + log.info("Starting " + userAgent);
> + }
> try {
> String url = link.getId();
> final Protocol protocol =
> droid.getProtocolFactory().getProtocol(url);
> openStream = protocol.openStream(url);
> if (protocol.isAllowed(url)) {
> String contentType = protocol.getContentType(url);
> - SimpleThreads.threadMessage("contentType " + contentType);
> + if (log.isInfoEnabled()) {
> + log.info("contentType " + contentType);
> + }
> if (contentType==null){
> - SimpleThreads.threadMessage("missing contentType... can't
> parse..." );
> + log.info("missing contentType... can't parse...");
> }
> else {
> Parser parser =
> droid.getParserFactory().getParser(contentType);
> if( parser == null ) {
> - SimpleThreads.threadMessage("could not find parser for:
> "+contentType);
> + if (log.isInfoEnabled()) {
> + log.info("could not find parser for: " + contentType);
> + }
> }
> else {
> Parse parse = parser.getParse(openStream, link);
> @@ -68,7 +78,7 @@
> }
> }
> else {
> - SimpleThreads.threadMessage("stopping processing since"
> + log.info("Stopping processing since"
> + " bots are not allowed for this url.");
> }
> }
>
> Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/
> walker/FileWorker.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/java/org/apache/droids/robot/walker/
> FileWorker.java (original)
> +++ incubator/droids/trunk/src/java/org/apache/droids/robot/walker/
> FileWorker.java Tue Nov 4 08:11:41 2008
> @@ -18,12 +18,12 @@
>
> import java.io.File;
>
> -import org.apache.droids.SimpleThreads;
> import org.apache.droids.api.TaskQueue;
> import org.apache.droids.api.Worker;
> import org.apache.droids.exception.InvalidTaskException;
> +import org.apache.droids.helper.Loggable;
>
> -public class FileWorker implements Worker<FileTask>
> +public class FileWorker extends Loggable implements Worker<FileTask>
> {
> final TaskQueue<FileTask> queue;
>
> @@ -47,7 +47,9 @@
> }
> }
> else {
> - SimpleThreads.threadMessage( "FILE: "+file.getAbsolutePath() );
> + if (log.isInfoEnabled()) {
> + log.info( "FILE: "+file.getAbsolutePath() );
> + }
> }
> }
> }
> \ No newline at end of file
>
> Modified: incubator/droids/trunk/src/test/java/org/apache/droids/
> localserver/LocalHttpServer.java
> URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/test/java/org/apache/droids/localserver/LocalHttpServer.java?rev=711289&r1=711288&r2=711289&view=diff
> =
> =
> =
> =
> =
> =
> =
> =
> ======================================================================
> --- incubator/droids/trunk/src/test/java/org/apache/droids/
> localserver/LocalHttpServer.java (original)
> +++ incubator/droids/trunk/src/test/java/org/apache/droids/
> localserver/LocalHttpServer.java Tue Nov 4 08:11:41 2008
> @@ -26,6 +26,7 @@
> import java.util.Set;
> import java.util.concurrent.atomic.AtomicInteger;
>
> +import org.apache.droids.helper.Loggable;
> import org.apache.http.ConnectionReuseStrategy;
> import org.apache.http.HttpException;
> import org.apache.http.HttpServerConnection;
> @@ -50,7 +51,7 @@
> /**
> * Local HTTP server for tests that require one.
> */
> -public class LocalHttpServer
> +public class LocalHttpServer extends Loggable
> {
>
> /**
> @@ -162,9 +163,8 @@
> try {
> this.servicedSocket.close();
> }
> - catch (IOException iox) {
> - System.out.println("error stopping " + this);
> - iox.printStackTrace(System.out);
> + catch (IOException ex) {
> + log.error(ex.getMessage(), ex);
> }
> finally {
> this.servicedSocket = null;
> @@ -237,11 +237,10 @@
> try {
> accept();
> }
> - catch (Exception e) {
> + catch (Exception ex) {
> ServerSocket ssock = servicedSocket;
> if ((ssock != null) && !ssock.isClosed()) {
> - System.out.println(LocalHttpServer.this.toString() +
> " could not accept");
> - e.printStackTrace(System.out);
> + log.error(LocalHttpServer.this.toString() + " could
> not accept", ex);
> }
> // otherwise ignore the exception silently
> break;
>
>