You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by ol...@apache.org on 2008/11/05 22:12:42 UTC
svn commit: r711718 - in /incubator/droids/trunk/src:
java/org/apache/droids/api/ java/org/apache/droids/handle/
java/org/apache/droids/helper/factories/ java/org/apache/droids/impl/
java/org/apache/droids/net/ java/org/apache/droids/robot/crawler/ jav...
Author: olegk
Date: Wed Nov 5 14:12:41 2008
New Revision: 711718
URL: http://svn.apache.org/viewvc?rev=711718&view=rev
Log:
DROIDS-28: improved exception handling
Added:
incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java
incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java
incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java
Modified:
incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java
incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java
incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java
incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java
incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java
incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java
incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java
incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java
incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java
incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java
incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java
incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java
incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java
incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java
incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
Modified: incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java Wed Nov 5 14:12:41 2008
@@ -40,7 +40,7 @@
*
* @throws DroidsException
*/
- void init();
+ void init() throws DroidsException;
/**
* Invoke an instance of the worker used in the droid
Modified: incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java Wed Nov 5 14:12:41 2008
@@ -16,9 +16,12 @@
*/
package org.apache.droids.api;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
+import org.apache.droids.exception.DroidsException;
+
/**
* A handler is a component that uses the stream, the parse and url to invoke
* arbitrary business logic on the objects.
@@ -36,5 +39,6 @@
* the parse object from a former processing step
* @throws Exception
*/
- void handle(InputStream openStream, URL url, Parse parse) throws Exception;
+ void handle(InputStream openStream, URL url, Parse parse)
+ throws IOException, DroidsException;
}
Added: incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java?rev=711718&view=auto
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java (added)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java Wed Nov 5 14:12:41 2008
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+public interface TaskExceptionHandler
+{
+
+ TaskExceptionResult handleException(Exception ex);
+
+}
Added: incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java?rev=711718&view=auto
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java (added)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java Wed Nov 5 14:12:41 2008
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+public enum TaskExceptionResult
+{
+
+ IGNORE, WARN, FATAL
+
+}
Modified: incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java Wed Nov 5 14:12:41 2008
@@ -16,6 +16,10 @@
*/
package org.apache.droids.api;
+import java.io.IOException;
+
+import org.apache.droids.exception.DroidsException;
+
/**
* A worker is the unit that is doing the actual work. A {@link Droid} is the
@@ -27,6 +31,6 @@
*/
public interface Worker<T extends Task> {
- void execute( final T task );
+ void execute( final T task ) throws DroidsException, IOException;
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java Wed Nov 5 14:12:41 2008
@@ -52,7 +52,7 @@
* java.net.URL, org.apache.droids.api.Parse)
*/
public void handle(InputStream stream, URL urlToHandle, Parse parse)
- throws Exception {
+ throws IOException {
this.url = urlToHandle;
writeOutput(stream);
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java Wed Nov 5 14:12:41 2008
@@ -48,7 +48,7 @@
* @see org.apache.droids.api.Handler#handle(java.io.InputStream,
* java.net.URL, org.apache.droids.api.Parse)
*/
- public void handle(InputStream stream, URL url, Parse parse) throws Exception {
+ public void handle(InputStream stream, URL url, Parse parse) throws IOException {
writeOutput(stream);
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java Wed Nov 5 14:12:41 2008
@@ -20,8 +20,7 @@
import java.io.Reader;
import java.io.Writer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.droids.helper.Loggable;
/**
* Wrapper that allows you to pipe a stream from a reader to a writer via a
@@ -30,11 +29,9 @@
* @version 1.0
*
*/
-public class WriterHandler {
+public class WriterHandler extends Loggable {
private static final int BUFFER_SIZE = 1024;
- protected final Log log = LogFactory.getLog(this.getClass()
- .getCanonicalName());
/**
* Pipes everything from the reader to the writer via a buffer
Modified: incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java Wed Nov 5 14:12:41 2008
@@ -18,8 +18,7 @@
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.droids.helper.Loggable;
/**
* Basically all factories till now extend this generic factory. The core is a
@@ -28,9 +27,7 @@
* @version 1.0
*
*/
-public class GenericFactory<T> {
-
- protected final Log log = LogFactory.getLog(this.getClass().getCanonicalName());
+public class GenericFactory<T> extends Loggable {
private Map<String,T> map = null;
Modified: incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java Wed Nov 5 14:12:41 2008
@@ -25,6 +25,7 @@
import org.apache.commons.io.IOUtils;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Parse;
+import org.apache.droids.exception.DroidsException;
/**
* Factory that will traverse all registered handler and execute them.
@@ -46,28 +47,21 @@
* the underlying parse object
* @return false if we found a problem, true if all went well
*/
- public boolean handle(InputStream stream, URL url, Parse parse) {
+ public boolean handle(InputStream stream, URL url, Parse parse)
+ throws DroidsException, IOException {
byte[] streamCopy = null;
if(stream==null){
return false;
}
- try {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copy(stream, out);
- streamCopy = out.toByteArray();
- } catch (IOException e) {
- log.fatal("Handler setup has thrown an error.", e);
- }
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copy(stream, out);
+ streamCopy = out.toByteArray();
for (Handler handler : getMap().values()) {
if (streamCopy == null) {
return false;
}
- try {
- handler.handle(new ByteArrayInputStream(streamCopy), url, parse);
- } catch (Exception e) {
- log.fatal("Handler \"" + handler + "\" has thrown an error.", e);
- }
+ handler.handle(new ByteArrayInputStream(streamCopy), url, parse);
}
return true;
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java Wed Nov 5 14:12:41 2008
@@ -38,7 +38,8 @@
* @return ready to use protocol plugin or null if non have been found
* @throws ProtocolNotFoundException
*/
- public Protocol getProtocol(String uri) throws ProtocolNotFoundException {
+ public Protocol getProtocol(String uri)
+ throws MalformedURLException, ProtocolNotFoundException {
URL url = null;
Protocol protocol = null;
try {
@@ -48,8 +49,6 @@
throw new ProtocolNotFoundException(uri);
}
protocol = getMap().get(protocolName);
- } catch (MalformedURLException e) {
- throw new ProtocolNotFoundException(uri, e.toString());
} catch (ProtocolNotFoundException e) {
throw new ProtocolNotFoundException(uri, e.toString());
}
Added: incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java?rev=711718&view=auto
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java (added)
+++ incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java Wed Nov 5 14:12:41 2008
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.impl;
+
+import java.io.IOException;
+
+import org.apache.droids.api.TaskExceptionHandler;
+import org.apache.droids.api.TaskExceptionResult;
+import org.apache.droids.exception.DroidsException;
+
+public class DefaultTaskExceptionHandler implements TaskExceptionHandler {
+
+ public TaskExceptionResult handleException(Exception ex) {
+ if (ex instanceof IOException) {
+ return TaskExceptionResult.WARN;
+ } else if (ex instanceof DroidsException) {
+ return TaskExceptionResult.WARN;
+ } else {
+ return TaskExceptionResult.FATAL;
+ }
+ }
+
+}
Modified: incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java Wed Nov 5 14:12:41 2008
@@ -25,6 +25,8 @@
import org.apache.droids.api.DelayTimer;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Task;
+import org.apache.droids.api.TaskExceptionHandler;
+import org.apache.droids.api.TaskExceptionResult;
import org.apache.droids.api.TaskMaster;
import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
@@ -46,6 +48,8 @@
private Date finishedWorking = null;
private int completedTask = 0;
private T lastCompletedTask = null;
+
+ private TaskExceptionHandler exHandler;
/**
* The queue has been initialized
@@ -80,7 +84,27 @@
if (log.isInfoEnabled()) {
log.info("Worker \"" + id + "\" has finished.");
}
- if (runningWorker.size() == 0 & !queue.hasNext()) {
+
+ boolean terminate = false;
+
+ Exception ex = worker.getException();
+ if (ex != null) {
+ TaskExceptionResult result = TaskExceptionResult.WARN;
+ if (exHandler != null) {
+ result = exHandler.handleException(ex);
+ }
+ switch (result) {
+ case WARN:
+ log.warn(ex.getMessage());
+ break;
+ case FATAL:
+ log.warn(ex.getMessage());
+ terminate = true;
+ break;
+ }
+ }
+
+ if (terminate || (runningWorker.size() == 0 & !queue.hasNext())) {
shutdownAndAwaitTermination();
long elapsed = System.currentTimeMillis() - startedWorking.getTime();
@@ -115,22 +139,21 @@
if( activeWorkers >= getMaxThreads() ) {
return; // don't make a new runner...
}
-
- try {
- WorkerRunner runner = new WorkerRunner();
- runningWorker.put(runner.getId(), runner);
- pool.execute(runner);
- if (log.isInfoEnabled()) {
- log.info("starting " + runner.getId());
- }
- }
- catch (Exception ex) {
- log.error(ex.getMessage(), ex);
- }
+ WorkerRunner runner = new WorkerRunner();
+ runningWorker.put(runner.getId(), runner);
+ pool.execute(runner);
+ if (log.isInfoEnabled()) {
+ log.info("starting " + runner.getId());
+ }
}
}
+ public final void setExceptionHandler(TaskExceptionHandler exHandler) {
+ this.exHandler = exHandler;
+ }
+
+
public DelayTimer getDelayTimer() {
return delayTimer;
}
@@ -189,6 +212,7 @@
class WorkerRunner extends Thread {
T task;
+ Exception exception;
@Override
public void run() {
@@ -206,13 +230,22 @@
}
Worker<T> worker = droid.getNewWorker();
- worker.execute( task );
+ try {
+ worker.execute( task );
+ } catch (Exception ex) {
+ exception = ex;
+ }
}
}
finally {
finishedWorker( getId() );
}
}
+
+ public Exception getException() {
+ return exception;
+ }
+
}
public int getCompletedTasks() {
Modified: incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java Wed Nov 5 14:12:41 2008
@@ -58,31 +58,17 @@
/**
* @param file
*/
- public void setFile(String file) {
+ public void setFile(String file) throws IOException {
URL url = null;
- try {
- if (file.startsWith("classpath:/")) {
- url = this.getClass().getResource(
- file.substring("classpath:/".length() - 1));
- } else {
- url = (file.contains(":/") ? new URL(file) : new URL("file://" + file));
- }
- log.debug("url " + url);
- Reader reader = null;
- try {
- reader = new InputStreamReader(url.openStream());
- } catch (RuntimeException e) {
- log.fatal("Can't create reader for url: " + url);
- }
- if (reader != null) {
- rules = readRulesFile(reader);
- } else {
- log.fatal("Can't find resource: " + file);
- }
- } catch (Exception e1) {
- log.fatal("File " + file + " has thrown an exception:\n" + e1);
+ if (file.startsWith("classpath:/")) {
+ url = this.getClass().getResource(
+ file.substring("classpath:/".length() - 1));
+ } else {
+ url = (file.contains(":/") ? new URL(file) : new URL("file://" + file));
}
-
+ log.debug("url " + url);
+ Reader reader = new InputStreamReader(url.openStream());
+ rules = readRulesFile(reader);
}
private RegexRule[] readRulesFile(Reader reader) throws IOException {
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java Wed Nov 5 14:12:41 2008
@@ -17,12 +17,15 @@
package org.apache.droids.robot.crawler;
import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.concurrent.TimeUnit;
import org.apache.droids.AbstractDroid;
import org.apache.droids.LinkTask;
import org.apache.droids.api.*;
+import org.apache.droids.exception.InvalidTaskException;
import org.apache.droids.helper.factories.HandlerFactory;
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
@@ -48,17 +51,18 @@
this.initialLocations = initialLocations;
}
- public void init() {
+ public void init() throws InvalidTaskException {
if( initialLocations == null || initialLocations.isEmpty() ) {
- throw new RuntimeException( "WebCrawlerDroid requires at least one starting file" );
+ throw new IllegalStateException( "WebCrawlerDroid requires at least one starting file" );
}
- for( String uri : initialLocations ) {
+ for( String location : initialLocations ) {
+ URI uri;
try {
- queue.merge( new LinkTask( null, new URI(uri), 0 ) );
- }
- catch( Exception ex ) {
- throw new RuntimeException( ex );
+ uri = new URI(location);
+ } catch (URISyntaxException ex) {
+ throw new InvalidTaskException("Invalid lication: " + location);
}
+ queue.merge( new LinkTask( null, uri, 0 ) );
}
}
@@ -111,25 +115,21 @@
//------------------------------------------------------------------
//------------------------------------------------------------------
- public static void main( String[] args )
+ public static void main( String[] args ) throws Exception
{
- try {
- MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
- taskMaster.setMaxThreads( 3 );
-
- TaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-
- Collection<String> locations = new ArrayList<String>();
- locations.add( args[0] );
-
- CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
- simple.setInitialLocations( locations );
- simple.init();
- simple.start(); // TODO? perhaps start internally calls init()?
- }
- catch (Exception e) {
- e.printStackTrace();
- }
+ MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
+ taskMaster.setMaxThreads( 3 );
+
+ TaskQueue<Link> queue = new SimpleTaskQueue<Link>();
+
+ Collection<String> locations = new ArrayList<String>();
+ locations.add( args[0] );
+
+ CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
+ simple.setInitialLocations( locations );
+ simple.init();
+ simple.start(); // TODO? perhaps start internally calls init()?
+ simple.getTaskMaster().awaitTermination(0, TimeUnit.SECONDS);
}
}
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java Wed Nov 5 14:12:41 2008
@@ -18,7 +18,6 @@
import java.io.IOException;
import java.io.InputStream;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -29,6 +28,7 @@
import org.apache.droids.api.Parser;
import org.apache.droids.api.Protocol;
import org.apache.droids.api.Worker;
+import org.apache.droids.exception.DroidsException;
import org.apache.droids.helper.Loggable;
import org.apache.droids.helper.factories.URLFiltersFactory;
@@ -41,7 +41,7 @@
this.droid = droid;
}
- public void execute(Link link)
+ public void execute(Link link) throws DroidsException, IOException
{
InputStream openStream = null;
final String userAgent = this.getClass().getCanonicalName();
@@ -82,29 +82,22 @@
+ " bots are not allowed for this url.");
}
}
- catch (Exception e) {
- e.printStackTrace();
- }
finally{
try {
if (openStream != null) {
openStream.close();
}
- } catch (IOException e) {
- e.printStackTrace();
+ } catch (IOException ex) {
+ log.error("Error closing stream", ex);
}
}
}
- protected void handle( Parse parse, InputStream openStream, Link link )
+ protected void handle( Parse parse, InputStream openStream, Link link )
+ throws DroidsException, IOException
{
String url = link.getId();
- try {
- droid.getHandlerFactory().handle(openStream, new URL(url), parse);
- }
- catch (MalformedURLException e) {
- e.printStackTrace();
- }
+ droid.getHandlerFactory().handle(openStream, new URL(url), parse);
}
protected Collection<Link> getFilteredOutlinks( Parse parse )
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java Wed Nov 5 14:12:41 2008
@@ -32,18 +32,12 @@
this.queue = queue;
}
- public void execute(FileTask task)
+ public void execute(FileTask task) throws InvalidTaskException
{
File file = task.getFile();
if( file.isDirectory() ) {
- // Add every file in the directory to the Queue
- try {
- for( File f : file.listFiles() ) {
- queue.merge( new FileTask( f, task.getDepth()+1 ) );
- }
- }
- catch (InvalidTaskException e) {
- e.printStackTrace();
+ for( File f : file.listFiles() ) {
+ queue.merge( new FileTask( f, task.getDepth()+1 ) );
}
}
else {
Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java Wed Nov 5 14:12:41 2008
@@ -19,6 +19,7 @@
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.concurrent.TimeUnit;
import org.apache.droids.AbstractDroid;
import org.apache.droids.exception.InvalidTaskException;
@@ -39,23 +40,18 @@
this.initialFiles = initialFiles;
}
- public void init() {
+ public void init() throws InvalidTaskException {
if( initialFiles == null || initialFiles.isEmpty() ) {
- throw new RuntimeException( "FileSystemWalker requires at least one starting file" );
+ throw new IllegalStateException( "FileSystemWalker requires at least one starting file" );
}
- try {
- for( File file : initialFiles ) {
- queue.merge( new FileTask( file, 0 ) );
- }
- }
- catch (InvalidTaskException e) {
- e.printStackTrace();
+ for( File file : initialFiles ) {
+ queue.merge( new FileTask( file, 0 ) );
}
}
public void finished()
{
- System.out.println( "FINISHED!!!" );
+ log.info( "FINISHED!!!" );
}
public FileWorker getNewWorker() {
@@ -65,7 +61,7 @@
//------------------------------------------------------------------
//------------------------------------------------------------------
- public static void main( String[] args )
+ public static void main( String[] args ) throws Exception
{
MultiThreadedTaskMaster<FileTask> taskMaster = new MultiThreadedTaskMaster<FileTask>();
taskMaster.setMaxThreads( 3 );
@@ -79,6 +75,7 @@
simple.setInitialFiles( files );
simple.init();
simple.start(); // TODO? perhaps start internally calls init()?
+ simple.getTaskMaster().awaitTermination(0, TimeUnit.SECONDS);
}
}
Modified: incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java Wed Nov 5 14:12:41 2008
@@ -29,6 +29,7 @@
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.impl.DefaultTaskExceptionHandler;
import org.apache.droids.impl.MultiThreadedTaskMaster;
import org.apache.droids.impl.SimpleTaskQueue;
import org.apache.droids.net.RegexURLFilter;
@@ -99,6 +100,7 @@
MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
taskMaster.setMaxThreads( 5 );
taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
CrawlingDroid helloCrawler = new CrawlingDroid( simpleQueue, taskMaster );
helloCrawler.setFiltersFactory(filtersFactory);
Modified: incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java (original)
+++ incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java Wed Nov 5 14:12:41 2008
@@ -100,7 +100,7 @@
HandlerFactory handlerFactory = new HandlerFactory();
Handler defaultHandler = new Handler() {
- public void handle(InputStream openStream, URL url, Parse parse) throws Exception {
+ public void handle(InputStream openStream, URL url, Parse parse) {
visitedLinks.add(url);
}