You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by rf...@apache.org on 2011/12/02 02:56:18 UTC
svn commit: r1209339 -
/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
Author: rfrovarp
Date: Fri Dec 2 02:56:17 2011
New Revision: 1209339
URL: http://svn.apache.org/viewvc?rev=1209339&view=rev
Log:
An attempt at fixing DROIDS-159
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=1209339&r1=1209338&r2=1209339&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java Fri Dec 2 02:56:17 2011
@@ -28,6 +28,7 @@ import org.apache.droids.api.ManagedCont
import org.apache.droids.api.Parse;
import org.apache.droids.api.Parser;
import org.apache.droids.api.Protocol;
+import org.apache.droids.api.Task;
import org.apache.droids.api.TaskValidator;
import org.apache.droids.api.Worker;
import org.apache.droids.exception.DroidsException;
@@ -93,7 +94,7 @@ public class CrawlingWorker implements W
}
else {
Parse parse = parser.parse(entity, link);
- if( parse.getOutlinks() != null && parse.isFollowed() ) {
+ if( parse.getNewTasks() != null && parse.isFollowed() ) {
Collection<Link> outlinks = getFilteredOutlinks( parse );
droid.getQueue().addAll( outlinks );
}
@@ -126,7 +127,12 @@ public class CrawlingWorker implements W
// TODO -- make the hashvalue for Outlink...
Map<String,Link> filtered = new LinkedHashMap<String,Link>();
- for( Link outlink : parse.getOutlinks() ) {
+ for( Task outTask : parse.getNewTasks() ) {
+ // only use Links, so if for some reason it isn't a Link, skip
+ if( !(outTask instanceof Link)) {
+ continue;
+ }
+ Link outlink = (Link)outTask;
String id = outlink.getId();
if (filters.accept(id) && !filtered.containsKey(id)) {
if( linkValidator == null ){