You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by ry...@apache.org on 2008/09/29 05:07:19 UTC
svn commit: r699967 - in /labs/droids/branch/LABS-144/src:
dynamics/java/org/apache/droids/droids-core-context.xml
examples/java/org/apache/droids/examples/SimpleRuntime.java
robots/java/org/apache/droids/crawler/CrawlingDroid.java
Author: ryan
Date: Sun Sep 28 20:07:19 2008
New Revision: 699967
URL: http://svn.apache.org/viewvc?rev=699967&view=rev
Log:
LABS-185 -- fixing some of the spring configuration...
NOTE, maxDepth and maxSize concepts still need to be added back...
<property name="maxDepth" value="${droids.queue.maxDepth}"/>
<property name="maxSize" value="${droids.queue.maxSize}"/>
Modified:
labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/droids-core-context.xml
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java
Modified: labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/droids-core-context.xml
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/droids-core-context.xml?rev=699967&r1=699966&r2=699967&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/droids-core-context.xml (original)
+++ labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/droids-core-context.xml Sun Sep 28 20:07:19 2008
@@ -39,20 +39,38 @@
<property name="locations" value="classpath:org/apache/droids/droids-core.properties"/>
</bean>
- <!-- Droids -->
- <bean name="org.apache.droids.api.Droid/hello"
- class="org.apache.droids.HelloCrawler">
- <property name="core" ref="org.apache.droids.Core"/>
- <property name="queue" ref="org.apache.droids.queue.Simple"/>
+
+ <bean name="taskmaster"
+ class="org.apache.droids.impl.MultiThreadedTaskMaster">
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
<property name="maxThreads" value="${droids.maxThreads}"/>
- <property name="url" value="${droids.initial.url}"/>
+ </bean>
+
+ <!-- Droids -->
+ <bean name="org.apache.droids.api.Droid/hello"
+ class="org.apache.droids.crawler.CrawlingDroid">
+ <constructor-arg ref="taskMaster" />
+ <constructor-arg ref="org.apache.droids.impl.SimpleTaskQueue" />
+
+ <property name="protocolFactory" ref="org.apache.droids.helper.factories.ProtocolFactory"/>
+ <property name="parserFactory" ref="org.apache.droids.helper.factories.ParserFactory"/>
+ <property name="handlerFactory" ref="org.apache.droids.helper.factories.HandlerFactory"/>
+ <property name="filtersFactory" ref="org.apache.droids.helper.factories.FiltersFactory"/>
+
+ <property name="initialLocations">
+ <set>
+ <ref bean="${droids.initial.url}" />
+ </set>
+ </property>
+
</bean>
<!-- Queue -->
- <bean id="org.apache.droids.queue.Simple"
- class="org.apache.droids.queue.Simple">
+ <bean id="org.apache.droids.impl.SimpleTaskQueue"
+ class="org.apache.droids.impl.SimpleTaskQueue">
+ <!--
<property name="maxDepth" value="${droids.queue.maxDepth}"/>
<property name="maxSize" value="${droids.queue.maxSize}"/>
+ -->
</bean>
<!-- Protocol -->
<bean
Modified: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java?rev=699967&r1=699966&r2=699967&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java Sun Sep 28 20:07:19 2008
@@ -106,8 +106,8 @@
helloCrawler.setProtocolFactory(protocolFactory);
helloCrawler.setHandlerFactory(handlerFactory);
- Collection<URI> initialLocations = new ArrayList<URI>();
- initialLocations.add( new URI( targetURL ) );
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURL );
helloCrawler.setInitialLocations(initialLocations);
helloCrawler.init();
Modified: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java?rev=699967&r1=699966&r2=699967&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java (original)
+++ labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java Sun Sep 28 20:07:19 2008
@@ -34,7 +34,7 @@
private final TaskQueue queue;
private final TaskMaster taskMaster;
- private Collection<URI> initialLocations;
+ private Collection<String> initialLocations;
ProtocolFactory protocolFactory;
ParserFactory parserFactory;
@@ -47,7 +47,7 @@
this.taskMaster = taskMaster;
}
- public void setInitialLocations(Collection<URI> initialLocations) {
+ public void setInitialLocations(Collection<String> initialLocations) {
this.initialLocations = initialLocations;
}
@@ -55,8 +55,13 @@
if( initialLocations == null || initialLocations.isEmpty() ) {
throw new RuntimeException( "WebCrawlerDroid requires at least one starting file" );
}
- for( URI uri : initialLocations ) {
- queue.merge( new LinkTask( null, uri, 0 ) );
+ for( String uri : initialLocations ) {
+ try {
+ queue.merge( new LinkTask( null, new URI(uri), 0 ) );
+ }
+ catch( Exception ex ) {
+ throw new RuntimeException( ex );
+ }
}
}
@@ -125,8 +130,8 @@
TaskQueue queue = new SimpleTaskQueue();
- Collection<URI> locations = new ArrayList<URI>();
- locations.add( new URI( args[0] ) );
+ Collection<String> locations = new ArrayList<String>();
+ locations.add( args[0] );
CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
simple.setInitialLocations( locations );
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org