You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by ry...@apache.org on 2008/09/29 04:46:44 UTC
svn commit: r699965 - in /labs/droids/branch/LABS-144/src:
core/java/org/apache/droids/ core/java/org/apache/droids/api/
core/java/org/apache/droids/impl/ core/java/org/apache/droids/queue/
core/java/org/apache/droidsX/api/ core/java/org/apache/droidsX...
Author: ryan
Date: Sun Sep 28 19:46:42 2008
New Revision: 699965
URL: http://svn.apache.org/viewvc?rev=699965&view=rev
Log:
LABS-185 -- massive refactoring to integrate new droid pattern (with TaskMaster)
* /src/dynamics NOT TESTED YET!!
Added:
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/LinkTask.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/OutlinkTask.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Outlink.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskMaster.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Queue.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/MultiThreadedTaskMaster.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/MultiThreadedTaskMaster.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueue.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/SimpleTaskQueue.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java
labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droids/Core.java
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenamerRacer.java
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerCrawler.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloCrawler.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloWorker.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/racer/
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileTask.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileTask.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileWorker.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWorker.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/WalkingDroid.java (contents, props changed)
- copied, changed from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWalker.java
Removed:
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/AbstractDroid.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/AbstractWorker.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/Core.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/Outlink.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/DelayWorker.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Queue.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/queue/
labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/api/
labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWalker.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWorker.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileTask.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/MultiThreadedTaskMaster.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/SimpleTaskQueue.java
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenamerRacer.java
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerCrawler.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloCrawler.java
labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloWorker.java
Modified:
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/ParseData.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Droid.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Link.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Parser.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Task.java
labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Worker.java
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerWorker.java
labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
labs/droids/branch/LABS-144/src/plugins/java/org/apache/droids/parse/html/HtmlParser.java
Added: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/LinkTask.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/LinkTask.java?rev=699965&view=auto
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/LinkTask.java (added)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/LinkTask.java Sun Sep 28 19:46:42 2008
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids;
+
+import java.io.File;
+import java.net.URI;
+import java.util.Date;
+
+import org.apache.droids.api.Link;
+import org.apache.droids.api.Task;
+
+public class LinkTask implements Link
+{
+ private final Date started;
+ private final int depth;
+ private final URI uri;
+ private final Link from;
+
+ private Date lastModifedDate;
+ private String[] linksTo;
+
+ public LinkTask( Link from, URI uri, int depth )
+ {
+ this.from = from;
+ this.uri = uri;
+ this.depth = depth;
+ this.started = new Date();
+ }
+
+ public String getId() {
+ return uri.toString();
+ }
+
+ public Date getTaskDate() {
+ return started;
+ }
+
+ public int getDepth() {
+ return depth;
+ }
+
+ public Link getFrom() {
+ return from;
+ }
+
+ public Date getLastModifiedDate() {
+ return lastModifedDate;
+ }
+
+ public String[] getTo() {
+ return linksTo;
+ }
+
+ public Date getLastModifedDate() {
+ return lastModifedDate;
+ }
+
+ public void setLastModifedDate(Date lastModifedDate) {
+ this.lastModifedDate = lastModifedDate;
+ }
+
+ public void setLinksTo(String[] linksTo) {
+ this.linksTo = linksTo;
+ }
+
+ public URI getURI() {
+ return uri;
+ }
+}
\ No newline at end of file
Added: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/OutlinkTask.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/OutlinkTask.java?rev=699965&view=auto
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/OutlinkTask.java (added)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/OutlinkTask.java Sun Sep 28 19:46:42 2008
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids;
+
+import java.net.URI;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.droids.api.Link;
+import org.apache.droids.api.Outlink;
+import org.apache.droids.api.Task;
+
+/**
+ * An outlink that implements the task interface.
+ *
+ * @version 1.0
+ *
+ */
+public class OutlinkTask extends LinkTask implements Outlink {
+ private final String anchor;
+
+ /**
+ * Create a new instance for the given parameters.
+ *
+ * @param toUrl
+ * the destination url
+ * @param anchor
+ * the anchor
+ * @param depth2
+ * the depth of the link
+ */
+ public OutlinkTask( Link from, URI uri, int depth, String text ) {
+ super( from, uri, depth );
+ this.anchor = text;
+ }
+
+ public OutlinkTask( Link from, URI uri, String text ) {
+ super( from, uri, from.getDepth()+1 );
+ this.anchor = text;
+ }
+
+ public String getAnchorText() {
+ return anchor;
+ }
+
+}
Modified: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/ParseData.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/ParseData.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/ParseData.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/ParseData.java Sun Sep 28 19:46:42 2008
@@ -16,6 +16,8 @@
*/
package org.apache.droids;
+import org.apache.droids.api.*;
+
/**
* The result object that are filled by a parser
*
Modified: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Droid.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Droid.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Droid.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Droid.java Sun Sep 28 19:46:42 2008
@@ -16,7 +16,6 @@
*/
package org.apache.droids.api;
-import org.apache.droids.Core;
import org.apache.droids.exception.DroidsException;
/**
@@ -29,12 +28,7 @@
*
* @version 1.0
*/
-public interface Droid extends Runnable {
- /**
- * Invoke an instance of the worker used in the droid
- * @see java.lang.Runnable#run()
- */
- void run();
+public interface Droid {
/**
* Initialize the queue. Can have different implementation but the main groups
@@ -46,38 +40,20 @@
*
* @throws DroidsException
*/
- void initQueue() throws DroidsException;
+ public void init();
/**
- * Which implementation of a queue are we using. Setter method to make sure
- * different implementation can override the method.
- * <p>
- * Not all implementations are using a queue at all.
- *
- * @param queue
- * queue implementation
- * @throws DroidsException
+ * Invoke an instance of the worker used in the droid
*/
- void setQueue(Queue queue) throws DroidsException;
+ public void start();
/**
- * Notification that we finished a given worker. The implementation would
- * normally remove the worker from the pool and start a new worker instead (if
- * needed). If not needed this method normally invokes/requests the shutdown
- * of the main process.
- *
- * @param id
- * the id of the worker that has finished
- * @throws DroidsException
+ * Invoke when the droid has completed
*/
- void finishedWorker(long id) throws DroidsException;
-
+ public void finished();
+
/**
- * Return the core configuration for the current Droid. The core contains all
- * configured objects ready to use.
- *
- * @return ready to use configured core
- * @throws DroidsException
+ * Ask the droid for a new worker
*/
- Core getCore() throws DroidsException;
+ public Worker getNewWorker();
}
Modified: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Link.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Link.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Link.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Link.java Sun Sep 28 19:46:42 2008
@@ -16,6 +16,10 @@
*/
package org.apache.droids.api;
+import java.net.URI;
+import java.util.Date;
+
+
/**
* Simple extension of a {@link Task}. Adding from/to link and the last
* modified attribute to the task object.
@@ -25,11 +29,16 @@
*/
public interface Link extends Task {
/**
+ * @return the URI to this link
+ */
+ URI getURI();
+
+ /**
* From where the link was created
*
* @return the parent link from where the link was coming from
*/
- String[] getFrom();
+ Link getFrom();
/**
* To where the link is pointing to
@@ -43,5 +52,5 @@
*
* @return last modified date
*/
- String getLastModifiedDate();
+ Date getLastModifiedDate();
}
Added: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Outlink.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Outlink.java?rev=699965&view=auto
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Outlink.java (added)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Outlink.java Sun Sep 28 19:46:42 2008
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+import java.net.URI;
+import java.util.Date;
+
+
+/**
+ * Simple extension of a {@link Task}. Adding from/to link and the last
+ * modified attribute to the task object.
+ *
+ * @version 1.0
+ *
+ */
+public interface Outlink extends Link {
+ String getAnchorText();
+}
Modified: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Parser.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Parser.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Parser.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Parser.java Sun Sep 28 19:46:42 2008
@@ -35,5 +35,5 @@
* the link that correspond to the stream
* @return the parse object
*/
- Parse getParse(InputStream openStream, Task link);
+ Parse getParse(InputStream openStream, Link link);
}
Modified: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Task.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Task.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Task.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Task.java Sun Sep 28 19:46:42 2008
@@ -16,6 +16,8 @@
*/
package org.apache.droids.api;
+import java.util.Date;
+
/**
* A task is a working instruction for a droid. One can limit the depth of the
* task. That is based on the fact that a droid can extract more tasks from the
@@ -33,26 +35,17 @@
* @return The id of the task
*/
String getId();
-
- /**
- * When did the task showed up the first time in the queue
- *
- * @return the date when the task registered with the queue
- */
- String getTaskDate();
-
+
/**
- * Which is the depth of the current task.
*
- * @return the depth of the current tas
+ * @return The depth of the task
*/
int getDepth();
/**
- * The limit of nested task extractions we want to allow
+ * When was the task created
*
- * @param depth
- * limit of nested task extractions
+ * @return the date when the task was created.
*/
- void setDepth(int depth);
+ Date getTaskDate();
}
Added: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskMaster.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskMaster.java?rev=699965&view=auto
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskMaster.java (added)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskMaster.java Sun Sep 28 19:46:42 2008
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+
+
+/**
+ * Responsible for running all the tasks
+ */
+public interface TaskMaster {
+ void processAllTasks( final TaskQueue queue, final Droid droid );
+}
Copied: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Queue.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java?p2=labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Queue.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Queue.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java Sun Sep 28 19:46:42 2008
@@ -16,6 +16,9 @@
*/
package org.apache.droids.api;
+import java.util.Collection;
+
+
/**
* A queue is the data structure where the different tasks are waiting for
* service.
@@ -26,7 +29,7 @@
* @version 1.0
*
*/
-public interface Queue {
+public interface TaskQueue {
/**
* Return the task that is identified with the given id
*
@@ -37,14 +40,6 @@
Task getTask(String id);
/**
- * Create the initial task list as queue
- *
- * @param initialTask
- * initial list of tasks
- */
- void init(Task[] initialTask);
-
- /**
* Do we have more task waiting for service
*
* @return true if we have. false otherwise
@@ -59,13 +54,20 @@
Task next();
/**
+ * Add a Task to the queue...
+ *
+ * @param task task to add to the Queue
+ */
+ void merge(Task task);
+
+ /**
* Merge a given list of tasks with the current queue. Whether or not we
* eliminate double entries is up to the implementation
*
- * @param filterLinks
+ * @param tasks
* the list of task that we want to add to the queue
*/
- void merge(Task[] filterLinks);
+ void merge(Collection<? extends Task> tasks);
/**
* How many task do we have <em>left</em> in the queue.
@@ -73,4 +75,4 @@
* @return number of tasks do we have <em>left</em> in the queue.
*/
int totalSize();
-}
+}
\ No newline at end of file
Propchange: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/TaskQueue.java
------------------------------------------------------------------------------
svn:mergeinfo =
Modified: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Worker.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Worker.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Worker.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/api/Worker.java Sun Sep 28 19:46:42 2008
@@ -16,7 +16,6 @@
*/
package org.apache.droids.api;
-import org.apache.droids.exception.DroidsException;
/**
* A worker is the unit that is doing the actual work. A {@link Droid} is the
@@ -26,34 +25,6 @@
* @version 1.0
*
*/
-public interface Worker extends Runnable {
- /**
- * @param queue
- * @throws DroidsException
- */
- void setQueue(Queue queue) throws DroidsException;
-
- /**
- * @param droid
- * @throws DroidsException
- */
- void setDroid(Droid droid) throws DroidsException;
-
- /**
- * @param x
- * @throws DroidsException
- */
- void setDepth(int x) throws DroidsException;
-
- /**
- * @return the configured maximal depth
- * @throws DroidsException
- */
- int getDepth() throws DroidsException;
-
- /**
- * @return the id of the worker
- * @throws DroidsException
- */
- long getId() throws DroidsException;
+public interface Worker<T extends Task> {
+ public void execute( final T task );
}
Copied: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/MultiThreadedTaskMaster.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/MultiThreadedTaskMaster.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/MultiThreadedTaskMaster.java?p2=labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/MultiThreadedTaskMaster.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/MultiThreadedTaskMaster.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/MultiThreadedTaskMaster.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/MultiThreadedTaskMaster.java Sun Sep 28 19:46:42 2008
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droidsX.impl;
+package org.apache.droids.impl;
import java.util.Date;
import java.util.concurrent.ConcurrentHashMap;
@@ -22,9 +22,8 @@
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.droids.Core;
import org.apache.droids.SimpleThreads;
-import org.apache.droidsX.api.*;
+import org.apache.droids.api.*;
public class MultiThreadedTaskMaster implements TaskMaster
{
@@ -110,7 +109,16 @@
}
}
}
-
+
+
+ public DelayTimer getDelayTimer() {
+ return delayTimer;
+ }
+
+
+ public void setDelayTimer(DelayTimer delayTimer) {
+ this.delayTimer = delayTimer;
+ }
/**
* Adjust number of allowed threads
Propchange: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueue.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/SimpleTaskQueue.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueue.java?p2=labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueue.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/SimpleTaskQueue.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/impl/SimpleTaskQueue.java (original)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueue.java Sun Sep 28 19:46:42 2008
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droidsX.impl;
+package org.apache.droids.impl;
import java.util.Collection;
import java.util.Map;
@@ -22,7 +22,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
-import org.apache.droidsX.api.*;
+import org.apache.droids.api.*;
/**
* A simple
@@ -72,7 +72,7 @@
/**
* Tasks with duplicate keys will be overwritten
*/
- public void merge(Collection<Task> tasks) {
+ public void merge(Collection<? extends Task> tasks) {
for( Task task : tasks ) {
merge( task );
}
@@ -85,4 +85,12 @@
}
return task;
}
+
+ public void clear()
+ {
+ synchronized( this ) {
+ queue.clear();
+ taskMap.clear();
+ }
+ }
}
Propchange: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueue.java
------------------------------------------------------------------------------
svn:mergeinfo =
Added: labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java?rev=699965&view=auto
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java (added)
+++ labs/droids/branch/LABS-144/src/core/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java Sun Sep 28 19:46:42 2008
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.impl;
+
+import java.util.Collections;
+import java.util.HashSet;
+
+import org.apache.droids.api.*;
+import org.apache.droids.api.Task;
+
+/**
+ * Extend the task queue to ignore any tasks we have already seen
+ */
+public class SimpleTaskQueueWithHistory extends SimpleTaskQueue {
+ private final java.util.Set<String> previous;
+
+ /**
+ * Simple queue constructor.
+ */
+ public SimpleTaskQueueWithHistory() {
+ super();
+ previous = Collections.synchronizedSet( new HashSet<String>() );
+ }
+
+ @Override
+ public void merge(Task task) {
+ // only add it if we have not seen the taskId before...
+ if( previous.add( task.getId() ) ) {
+ super.merge( task );
+ }
+ }
+
+ @Override
+ public void clear()
+ {
+ previous.clear();
+ }
+}
Copied: labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droids/Core.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java?p2=labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droids/Core.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droids/Core.java (original)
+++ labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java Sun Sep 28 19:46:42 2008
@@ -92,7 +92,9 @@
*/
public void start(String name){
Droid droid = getDroid(name);
- droid.run();
+
+ droid.init();
+ droid.start();
}
Propchange: labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/branch/LABS-144/src/dynamics/java/org/apache/droids/Core.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java (from r699956, labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenamerRacer.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java?p2=labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java&p1=labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenamerRacer.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenamerRacer.java (original)
+++ labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java Sun Sep 28 19:46:42 2008
@@ -1,50 +1,47 @@
package org.apache.droids.examples;
import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import org.apache.droids.AbstractDroid;
-import org.apache.droids.AbstractWorker;
-import org.apache.droids.api.Worker;
+import org.apache.droids.api.*;
+import org.apache.droids.walker.FileTask;
import org.apache.droids.exception.DroidsException;
-import org.apache.droids.queue.QueueLink;
-
-public class FileRenamerRacer extends AbstractDroid {
- private String url;
-
- /**
- * Set the initial url
- *
- * @param url
- * the initial url
- */
- public void setUrl(String url) {
- this.url = url;
- }
-
- @Override
- public Worker getWorker() throws DroidsException {
- RenameWorker worker = new RenameWorker();
- worker.setQueue(getQueue());
- worker.setDroid(this);
- return worker;
- }
-
- public void initQueue() throws DroidsException {
- File target = new File(url);
- if (target.exists() & target.isDirectory()) {
- File[] list = target.listFiles();
- LinkedHashSet<QueueLink> set = new LinkedHashSet<QueueLink>();
- for (File file : list) {
- QueueLink link = new QueueLink(file.getAbsolutePath(), AbstractDroid.getNewTaskDate(), 0);
- set.add(link);
- }
- QueueLink[] outlinks = new QueueLink[0];
- outlinks = set.toArray(new QueueLink[set.size()]);
- getQueue().init(outlinks);
+import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggerFactory;
+
+
+public class FileRenameDroid implements Droid
+{
+ static final Logger log = Logger.getLogger( FileRenameDroid.class );
+
+ private final TaskQueue queue;
+ private final TaskMaster taskMaster;
+
+ private Collection<File> initialFiles;
+
+ public FileRenameDroid( TaskQueue queue, TaskMaster taskMaster )
+ {
+ this.queue = queue;
+ this.taskMaster = taskMaster;
+ }
+
+ public void setInitialFiles(Collection<File> initialFiles) {
+ this.initialFiles = initialFiles;
+ }
+
+ public void init() {
+ if( initialFiles == null || initialFiles.isEmpty() ) {
+ throw new RuntimeException( "FileSystemWalker requires at least one starting file" );
+ }
+ for( File file : initialFiles ) {
+ queue.merge( new FileTask( file, 0 ) );
}
}
+
public LinkedHashMap<String, String> cleaner = null;
@@ -65,31 +62,40 @@
cleaner.put(".-.", ".");
cleaner.put(",", "");
}
+
+ public void start()
+ {
+ taskMaster.processAllTasks(queue, this);
+ }
+
+ public void finished()
+ {
+ System.out.println( "FINISHED!!!" );
+ }
+
+ public RenameWorker getNewWorker() {
+ return new RenameWorker();
+ }
- public class RenameWorker extends AbstractWorker implements Worker {
+
+ public class RenameWorker implements Worker<FileTask> {
String replace;
- @Override
- public void run() {
+ public void execute(FileTask task) {
+
for (String pattern : getCleaner().keySet() ) {
replace = getCleaner().get(pattern);
- cleanFileName(pattern, replace, getLink().getId());
- try {
- getDroid().finishedWorker(getId());
- } catch (DroidsException e) {
- log.fatal(e);
- }
-
+ cleanFileName(task.getFile(), pattern, replace);
}
}
-
- private void cleanFileName(String pattern, String replace, String fileName) {
- log.debug("Processing: " + fileName);
+
+ private void cleanFileName(File file, String pattern, String replace) {
+ log.debug("Processing: " + file.getName());
log.debug("finding pattern: " + pattern);
log.debug("replacing it with: " + replace);
- File file = new File(fileName);
+ String fileName = file.getName();
if (fileName.contains(pattern) || !fileName.toLowerCase().equals(fileName)) {
log.debug("need to process this file: " + fileName + " in "
+ file.getAbsolutePath());
@@ -98,10 +104,30 @@
+ fileName.replaceAll(pattern, replace).toLowerCase());
log.debug("Renaming to: " + replacement.getName() + " in "
+ replacement.getAbsolutePath());
- file.renameTo(replacement);
+
+ log.info( "TODO! actually do the rename!" );
+ // TODO -- actually do the rename...file.renameTo(replacement);
}
}
-
}
+ //------------------------------------------------------------------
+ //------------------------------------------------------------------
+
+ public static void main( String[] args )
+ {
+ MultiThreadedTaskMaster taskMaster = new MultiThreadedTaskMaster();
+ taskMaster.setMaxThreads( 3 );
+
+ TaskQueue queue = new SimpleTaskQueue();
+
+ Collection<File> files = new ArrayList<File>();
+ files.add( new File( args[0] ) );
+
+ FileRenameDroid simple = new FileRenameDroid( queue, taskMaster );
+ simple.setInitialFiles( files );
+ simple.init();
+ simple.start(); // TODO? perhaps start internally calls init()?
+ }
}
+
Propchange: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/FileRenameDroid.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java (from r699956, labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerCrawler.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java?p2=labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java&p1=labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerCrawler.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerCrawler.java (original)
+++ labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java Sun Sep 28 19:46:42 2008
@@ -5,51 +5,22 @@
import java.io.ByteArrayOutputStream;
import org.apache.droids.Core;
-import org.apache.droids.HelloCrawler;
import org.apache.droids.SimpleThreads;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
+import org.apache.droids.crawler.CrawlingDroid;
import org.apache.droids.exception.DroidsException;
import org.apache.droids.handle.Solr;
import org.apache.http.PostFile;
-public class IndexerCrawler extends HelloCrawler{
+public class IndexerDroid extends CrawlingDroid {
- private String updateUrl;
- /**
- * We want to use the indexerWorker since it limits
- * the handling to the SolrHandler
- * @return IndexerWorker
- * @throws DroidsException
- */
- @Override
- public synchronized Worker getWorker() throws DroidsException {
- IndexerWorker worker = new IndexerWorker();
- worker.setQueue(getQueue());
- worker.setDroid(this);
- return worker;
- }
- /**
- * Implementing this method since we want to send the commit
- * command on the end of the whole queue.
- */
- @Override
- protected synchronized void shutdownAndAwaitTermination() {
- super.shutdownAndAwaitTermination();
- ByteArrayOutputStream out;
- BufferedInputStream stream ;
- PostFile post;
- Solr solr = new Solr();
- solr.setUpdateUrl(updateUrl);
- try {
- out = solr.createCommitDocument();
- stream = new BufferedInputStream(
- new ByteArrayInputStream(out.toByteArray()));
- post = new PostFile(updateUrl, stream);
- } catch (Exception e) {
- SimpleThreads.threadMessage(e.getMessage());
- }
-
+ public IndexerDroid(TaskQueue queue, TaskMaster taskMaster) {
+ super(queue, taskMaster);
}
+
+ private String updateUrl;
public String getUpdateUrl() {
return updateUrl;
Propchange: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerDroid.java
------------------------------------------------------------------------------
svn:mergeinfo =
Modified: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerWorker.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerWorker.java (original)
+++ labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/IndexerWorker.java Sun Sep 28 19:46:42 2008
@@ -1,30 +1,45 @@
package org.apache.droids.examples;
import java.io.IOException;
+import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.droids.Core;
-import org.apache.droids.HelloWorker;
import org.apache.droids.SimpleThreads;
import org.apache.droids.api.Handler;
+import org.apache.droids.api.Link;
import org.apache.droids.api.Parse;
+import org.apache.droids.crawler.CrawlingDroid;
+import org.apache.droids.crawler.CrawlingWorker;
import org.apache.droids.exception.DroidsException;
-public class IndexerWorker extends HelloWorker {
+public class IndexerWorker extends CrawlingWorker {
+
+ final Handler solrHandler;
+
+ public IndexerWorker(IndexerDroid droid, Handler handler) {
+ super(droid);
+
+ solrHandler = handler;
+ }
+
+
@Override
- protected void handle(Parse parse) throws MalformedURLException, IOException {
- Handler handler = getHandlerFactory().resolve("solr");
- try {
- handler.handle(getProtocol().openStream(getUri()), new URL(getUri()), parse);
- } catch (Exception e) {
- SimpleThreads.threadMessage(e.getMessage());
- }
- try {
- getDroid().finishedWorker(super.getId());
- } catch (DroidsException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
+ protected void handle( Parse parse, InputStream openStream, Link link )
+ {
+ // TODO -- something different...
+// Handler handler = getHandlerFactory().resolve("solr");
+// try {
+// handler.handle(getProtocol().openStream(getUri()), new URL(getUri()), parse);
+// } catch (Exception e) {
+// SimpleThreads.threadMessage(e.getMessage());
+// }
+// try {
+// getDroid().finishedWorker(super.getId());
+// } catch (DroidsException e) {
+// // TODO Auto-generated catch block
+// e.printStackTrace();
+// }
}
}
Modified: labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ labs/droids/branch/LABS-144/src/examples/java/org/apache/droids/examples/SimpleRuntime.java Sun Sep 28 19:46:42 2008
@@ -16,10 +16,13 @@
*/
package org.apache.droids.examples;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import org.apache.droids.Core;
-import org.apache.droids.HelloCrawler;
+import org.apache.droids.crawler.CrawlingDroid;
import org.apache.droids.delay.SimpleDelayTimer;
import org.apache.droids.exception.DroidsException;
import org.apache.droids.handle.Sysout;
@@ -28,10 +31,11 @@
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.SimpleTaskQueue;
import org.apache.droids.net.RegexURLFilter;
import org.apache.droids.parse.html.HtmlParser;
import org.apache.droids.protocol.http.Http;
-import org.apache.droids.queue.Simple;
/**
* Simple Droids runtime that wires various components together in Java code
@@ -43,7 +47,7 @@
private SimpleRuntime(){
}
- public static void main(String[] args) throws DroidsException {
+ public static void main(String[] args) throws Exception {
if (args.length < 1) {
System.out.println("Please specify a URL to crawl");
@@ -86,34 +90,30 @@
DroidFactory droidFactory = new DroidFactory();
droidFactory.setMap(new HashMap<String, Object>());
- // Assemble Droid Core
- Core core = new Core();
- core.setDroids(droidFactory);
- core.setParserFactory(parserFactory);
- core.setProtocolFactory(protocolFactory);
- core.setFiltersFactory(filtersFactory);
- core.setHandlerFactory(handlerFactory);
-
// Create default droid
SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
simpleDelayTimer.setDelay(100);
- Simple simpleQueue = new Simple();
- simpleQueue.setMaxDepth(3);
- simpleQueue.setMaxSize(-1);
-
- HelloCrawler helloCrawler = new HelloCrawler();
- helloCrawler.setCore(core);
- helloCrawler.setDelayTimer(simpleDelayTimer);
- helloCrawler.setQueue(simpleQueue);
- helloCrawler.setMaxThreads(5);
- helloCrawler.setUrl(targetURL);
-
- // Add default droid to the droid factory
- droidFactory.getMap().put("hello", helloCrawler);
+ SimpleTaskQueue simpleQueue = new SimpleTaskQueue();
+ // simpleQueue.setMaxDepth(3);
+ // simpleQueue.setMaxSize(-1);
+
+ MultiThreadedTaskMaster taskMaster = new MultiThreadedTaskMaster();
+ taskMaster.setMaxThreads( 5 );
+ taskMaster.setDelayTimer( simpleDelayTimer );
+
+ CrawlingDroid helloCrawler = new CrawlingDroid( simpleQueue, taskMaster );
+ helloCrawler.setFiltersFactory(filtersFactory);
+ helloCrawler.setParserFactory(parserFactory);
+ helloCrawler.setProtocolFactory(protocolFactory);
+ helloCrawler.setHandlerFactory(handlerFactory);
+
+ Collection<URI> initialLocations = new ArrayList<URI>();
+ initialLocations.add( new URI( targetURL ) );
+ helloCrawler.setInitialLocations(initialLocations);
- // Start
- core.start("hello");
+ helloCrawler.init();
+ helloCrawler.start();
}
}
Modified: labs/droids/branch/LABS-144/src/plugins/java/org/apache/droids/parse/html/HtmlParser.java
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/plugins/java/org/apache/droids/parse/html/HtmlParser.java?rev=699965&r1=699964&r2=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/plugins/java/org/apache/droids/parse/html/HtmlParser.java (original)
+++ labs/droids/branch/LABS-144/src/plugins/java/org/apache/droids/parse/html/HtmlParser.java Sun Sep 28 19:46:42 2008
@@ -18,17 +18,20 @@
import java.io.InputStream;
import java.net.MalformedURLException;
+import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import org.apache.droids.api.Link;
import org.apache.droids.api.Parse;
import org.apache.droids.api.Parser;
import org.apache.droids.api.Task;
import org.apache.droids.helper.Loggable;
-import org.apache.droids.Outlink;
+import org.apache.droids.api.Outlink;
+import org.apache.droids.OutlinkTask;
import org.apache.droids.ParseData;
import org.apache.droids.parse.ParseImpl;
import org.apache.html.dom.HTMLDocumentImpl;
@@ -63,9 +66,9 @@
private URL base = null;
- private Task link = null;
+ private Link link = null;
- public Parse getParse(InputStream stream, Task newLink) {
+ public Parse getParse(InputStream stream, Link newLink) {
this.link = newLink;
try {
this.base = new URL(newLink.getId());
@@ -161,13 +164,15 @@
newUrl=new URL(base, target).toString();
}
if (!newUrl.equals("")) {
- final Outlink outlink = new Outlink(
- target.contains(":/") ? target : newUrl, link.getDepth() + 1);
+ // Link from, URI uri, int depth, String text
+ String url = target.contains(":/") ? target : newUrl;
+ URI uri = new URI( url );
+ final OutlinkTask outlink = new OutlinkTask( link, uri, null );
log.debug("set size: "+set.size());
- log.debug("outlink.getToUrl(): "+outlink.getToUrl());
- log.debug("set.contains(outlink.getToUrl(): "+set.contains(outlink.getToUrl()));
- if (!set.contains(outlink.getToUrl())) {
- set.add(outlink.getToUrl());
+ log.debug("outlink.getToUrl(): "+outlink.getURI());
+ log.debug("set.contains(outlink.getToUrl(): "+set.contains(url));
+ if (!set.contains(url)) {
+ set.add(url);
links.add(outlink);
}
}
Copied: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java (from r699956, labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloCrawler.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java?p2=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java&p1=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloCrawler.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloCrawler.java (original)
+++ labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java Sun Sep 28 19:46:42 2008
@@ -14,78 +14,130 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droids;
+package org.apache.droids.crawler;
-import org.apache.droids.api.DelayTimer;
-import org.apache.droids.api.DelayWorker;
-import org.apache.droids.api.Task;
-import org.apache.droids.api.Worker;
-import org.apache.droids.exception.DroidsException;
-import org.apache.droids.queue.QueueLink;
-
-/**
- * Default implementation of a crawler.
- * The crawler needs a starting url and
- * extract then the next links with the
- * help of the defaultWorker.
- *
- * Extend this method for your own Droid.
- * @version 1.0
- */
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.droids.LinkTask;
+import org.apache.droids.api.*;
+import org.apache.droids.helper.factories.HandlerFactory;
+import org.apache.droids.helper.factories.ParserFactory;
+import org.apache.droids.helper.factories.ProtocolFactory;
+import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.SimpleTaskQueue;
+
+public class CrawlingDroid implements Droid
+{
+ private final TaskQueue queue;
+ private final TaskMaster taskMaster;
+
+ private Collection<URI> initialLocations;
-public class HelloCrawler extends AbstractDroid {
+ ProtocolFactory protocolFactory;
+ ParserFactory parserFactory;
+ HandlerFactory handlerFactory;
+ URLFiltersFactory filtersFactory;
+
+ public CrawlingDroid( TaskQueue queue, TaskMaster taskMaster )
+ {
+ this.queue = queue;
+ this.taskMaster = taskMaster;
+ }
- /*
- * Start the queue.
- * @see org.apache.droids.api.Droid#initQueue()
- */
- public void initQueue() throws DroidsException{
- synchronized (this) {
- QueueLink initialLink = new QueueLink(url, AbstractDroid.getNewTaskDate(), 0);
- getQueue().init(new Task[] { initialLink });
+ public void setInitialLocations(Collection<URI> initialLocations) {
+ this.initialLocations = initialLocations;
+ }
+
+ public void init() {
+ if( initialLocations == null || initialLocations.isEmpty() ) {
+ throw new RuntimeException( "WebCrawlerDroid requires at least one starting file" );
+ }
+ for( URI uri : initialLocations ) {
+ queue.merge( new LinkTask( null, uri, 0 ) );
}
}
- private String url = "";
- /**
- * Return the initial url
- * @return the initial url
- */
- public String getUrl() {
- return url;
- }
-
- /**
- * Set the initial url
- * @param url the initial url
- */
- public void setUrl(String url) {
- this.url = url;
- }
-
- private DelayTimer delayTimer;
-
- /**
- * Get the DelayTimer implementation that we want to use.
- * @return the delayTimer we use in the droid
- */
- public DelayTimer getDelayTimer() {
- return delayTimer;
- }
-
- public void setDelayTimer(DelayTimer delayTimer) throws DroidsException {
- this.delayTimer = delayTimer;
- }
-
- /*
- * @see org.apache.droids.AbstractDroid#getWorker()
- */
- @Override
- public Worker getWorker() throws DroidsException {
- DelayWorker worker = new HelloWorker();
- worker.setQueue(getQueue());
- worker.setDroid(this);
- worker.setDelayTimer(getDelayTimer());
- return worker;
+ public void start()
+ {
+ taskMaster.processAllTasks(queue, this);
+ }
+
+ public void finished()
+ {
+ System.out.println( "FINISHED!!!" );
+ }
+
+ public CrawlingWorker getNewWorker() {
+ return new CrawlingWorker( this );
+ }
+
+ public ProtocolFactory getProtocolFactory() {
+ return protocolFactory;
+ }
+
+ public void setProtocolFactory(ProtocolFactory protocolFactory) {
+ this.protocolFactory = protocolFactory;
+ }
+
+ public ParserFactory getParserFactory() {
+ return parserFactory;
+ }
+
+ public void setParserFactory(ParserFactory parserFactory) {
+ this.parserFactory = parserFactory;
+ }
+
+ public HandlerFactory getHandlerFactory() {
+ return handlerFactory;
+ }
+
+ public void setHandlerFactory(HandlerFactory handlerFactory) {
+ this.handlerFactory = handlerFactory;
+ }
+
+ public TaskQueue getQueue() {
+ return queue;
+ }
+
+ public TaskMaster getTaskMaster() {
+ return taskMaster;
+ }
+
+ public URLFiltersFactory getFiltersFactory() {
+ return filtersFactory;
+ }
+
+ public void setFiltersFactory(URLFiltersFactory filtersFactory) {
+ this.filtersFactory = filtersFactory;
+ }
+
+ //------------------------------------------------------------------
+ //------------------------------------------------------------------
+
+ public static void main( String[] args )
+ {
+ try {
+ MultiThreadedTaskMaster taskMaster = new MultiThreadedTaskMaster();
+ taskMaster.setMaxThreads( 3 );
+
+ TaskQueue queue = new SimpleTaskQueue();
+
+ Collection<URI> locations = new ArrayList<URI>();
+ locations.add( new URI( args[0] ) );
+
+ CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
+ simple.setInitialLocations( locations );
+ simple.init();
+ simple.start(); // TODO? perhaps start internally calls init()?
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
}
}
+
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingDroid.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java (from r699956, labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloWorker.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java?p2=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java&p1=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloWorker.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/HelloWorker.java (original)
+++ labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java Sun Sep 28 19:46:42 2008
@@ -14,99 +14,72 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droids;
+package org.apache.droids.crawler;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.droids.api.DelayTimer;
-import org.apache.droids.api.Droid;
-import org.apache.droids.api.Parse;
-import org.apache.droids.api.Parser;
-import org.apache.droids.api.Protocol;
-import org.apache.droids.api.Queue;
-import org.apache.droids.api.Task;
-import org.apache.droids.api.DelayWorker;
-import org.apache.droids.exception.DroidsException;
+import org.apache.droids.SimpleThreads;
+import org.apache.droids.api.*;
import org.apache.droids.helper.factories.HandlerFactory;
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
-import org.apache.droids.Outlink;
+import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.SimpleTaskQueue;
-public class HelloWorker extends Thread implements DelayWorker {
-
- protected final Log log = LogFactory.getLog(this.getClass().getCanonicalName());
-
- private Queue queue;
-
- private Droid droid;
-
- private Task link;
-
- private ProtocolFactory protocolFactory;
-
- protected int id;
-
- private ParserFactory parserFactory;
-
- private Parser parser;
-
- private Protocol protocol;
-
- private String uri;
-
- private URLFiltersFactory filtersFactory;
-
- private HandlerFactory handlerFactory;
-
- private DelayTimer delayTimer;
+public class CrawlingWorker implements Worker<Link>
+{
+ private final CrawlingDroid droid;
- private InputStream openStream;
-
- private int depth;
+ public CrawlingWorker( CrawlingDroid droid )
+ {
+ this.droid = droid;
+ }
- @Override
- public void run() {
+ public void execute(Link link)
+ {
+ InputStream openStream = null;
final String userAgent = this.getClass().getCanonicalName();
SimpleThreads.threadMessage("Starting " + userAgent);
try {
- if(delayTimer.getDelayMillis() > 0)
- {
- log.debug("Delay " + delayTimer.getDelayMillis() + "ms");
- sleep(delayTimer.getDelayMillis());
- }
- setUri(link.getId());
- String url = getUri();
- SimpleThreads.threadMessage("url " + url);
- final Protocol protocol = protocolFactory.getProtocol(url);
- setProtocol(protocol);
- openStream = getProtocol().openStream(getUri());
- long workerId = getId();
+ String url = link.getId();
+ final Protocol protocol = droid.getProtocolFactory().getProtocol(url);
+ openStream = protocol.openStream(url);
if (protocol.isAllowed(url)) {
String contentType = protocol.getContentType(url);
SimpleThreads.threadMessage("contentType " + contentType);
- if (contentType!=null){
- parser = parserFactory.getParser(contentType);
+ if (contentType==null){
+ SimpleThreads.threadMessage("missing contentType... can't parse..." );
}
- // parse contains the outlinks and can be used later
- Parse parse = getParse();
- handle(parse);
- //SimpleThreads.threadMessage("Trying to shut down "+getId());
- getDroid().finishedWorker(workerId);
- } else {
+ else {
+ Parser parser = droid.getParserFactory().getParser(contentType);
+ Parse parse = parser.getParse(openStream, link);
+ if( parse.getData() != null ) {
+ Collection<Outlink> outlinks = getFilteredOutlinks( parse );
+ droid.getQueue().merge( outlinks );
+ }
+ handle( parse, openStream, link );
+ }
+ }
+ else {
SimpleThreads.threadMessage("stopping processing since"
+ " bots are not allowed for this url.");
- getDroid().finishedWorker(workerId);
}
- } catch (Exception e) {
+ }
+ catch (Exception e) {
e.printStackTrace();
- }finally{
+ }
+ finally{
try {
if (openStream != null) {
openStream.close();
@@ -116,142 +89,32 @@
}
}
}
-
- protected void handle(Parse parse) throws MalformedURLException, IOException {
- String url = getUri();
- handlerFactory.handle(openStream, new URL(url), parse);
- }
-
- protected Parse getParse() {
- Parse parse = null;
- if (null != parser) {
- try {
- // extract links
- parse = parser.getParse(openStream, link);
- // all links from the page unfiltered
- filter(parse);
- } catch (Exception e) {
- SimpleThreads.threadMessage(e.getMessage());
- }
- }
- return parse;
- }
-
- protected void filter(Parse parse) {
- Outlink[] filterLinks = filterLinks(parse);
- getQueue().merge(filterLinks);
- }
-
- protected Outlink[] filterLinks(Parse parse) {
- synchronized (this) {
- // filter the link
- Outlink[] links = parse.getData().getOutlinks();
- // new cleaned list
- ArrayList<Outlink> filtered = new ArrayList<Outlink>();
- for (int i = 0; i < links.length; i++) {
- Outlink outlink = links[i];
- if (filtersFactory.accept(outlink.getToUrl()) && !filtered.contains(outlink)) {
- filtered.add(outlink);
- }
- }
- // this are the links we need to follow
- Outlink[] filterLinks = filtered.toArray(new Outlink[filtered.size()]);
- return filterLinks;
- }
- }
-
- public void setQueue(Queue queue) {
- synchronized (this) {
- this.queue = queue;
- link = getQueue().next();
- if (link != null) {
- depth = link.getDepth();
- }
- }
- }
-
- public void setDroid(Droid droid) throws DroidsException {
- this.droid = droid;
- protocolFactory = droid.getCore().getProtocolFactory();
- parserFactory = droid.getCore().getParserFactory();
- filtersFactory = droid.getCore().getFiltersFactory();
- handlerFactory = droid.getCore().getHandlerFactory();
- }
-
- public int getDepth() {
- return depth;
- }
-
- @Override
- public long getId() {
- return super.getId();
- }
- public void setDepth(int x) {
- depth = x;
- }
-
- public Task getLink() {
- return link;
- }
-
- public ProtocolFactory getProtocolFactory() {
- return protocolFactory;
- }
-
- public ParserFactory getParserFactory() {
- return parserFactory;
- }
-
- public URLFiltersFactory getFiltersFactory() {
- return filtersFactory;
- }
-
- public HandlerFactory getHandlerFactory() {
- return handlerFactory;
- }
-
- public Droid getDroid() {
- synchronized (this) {
- return droid;
- }
- }
-
- public void setUri(String uri) {
- synchronized (this) {
- this.uri = uri;
- }
- }
-
- public String getUri() {
- synchronized (this) {
- return uri;
- }
- }
-
- public void setProtocol(Protocol protocol) {
- synchronized (this) {
- this.protocol = protocol;
- }
- }
-
- public Protocol getProtocol() {
- synchronized (this) {
- return protocol;
+ protected void handle( Parse parse, InputStream openStream, Link link )
+ {
+ String url = link.getId();
+ try {
+ droid.getHandlerFactory().handle(openStream, new URL(url), parse);
+ }
+ catch (MalformedURLException e) {
+ e.printStackTrace();
}
}
-
- public Queue getQueue() {
- synchronized (this) {
- return queue;
+
+ protected Collection<Outlink> getFilteredOutlinks( Parse parse )
+ {
+ Outlink[] links = parse.getData().getOutlinks();
+ // new cleaned list
+ URLFiltersFactory filters = droid.getFiltersFactory();
+ // TODO -- make the hashvalue for Outlink...
+ Map<String,Outlink> filtered = new HashMap<String,Outlink>();
+ for( Outlink outlink : links ) {
+ String id = outlink.getId();
+ if (filters.accept(outlink.getId()) && !filtered.containsKey(id)) {
+ filtered.put(id,outlink);
+ }
}
+ return filtered.values();
}
+}
- public DelayTimer getDelayTimer() {
- return delayTimer;
- }
-
- public void setDelayTimer(DelayTimer delayTimer) throws DroidsException {
- this.delayTimer = delayTimer;
- }
-}
\ No newline at end of file
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/crawler/CrawlingWorker.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileTask.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileTask.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileTask.java?p2=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileTask.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileTask.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileTask.java (original)
+++ labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileTask.java Sun Sep 28 19:46:42 2008
@@ -14,12 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droidsX.depot.filesystem;
+package org.apache.droids.walker;
import java.io.File;
import java.util.Date;
-import org.apache.droidsX.api.Task;
+import org.apache.droids.api.Task;
+
public class FileTask implements Task
{
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileTask.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileWorker.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWorker.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileWorker.java?p2=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileWorker.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWorker.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWorker.java (original)
+++ labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileWorker.java Sun Sep 28 19:46:42 2008
@@ -14,20 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droidsX.depot.filesystem;
+package org.apache.droids.walker;
import java.io.File;
-import org.apache.droids.Core;
import org.apache.droids.SimpleThreads;
-import org.apache.droidsX.api.TaskQueue;
-import org.apache.droidsX.api.Worker;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
-public class FileSystemWorker implements Worker<FileTask>
+public class FileWorker implements Worker<FileTask>
{
final TaskQueue queue;
- public FileSystemWorker( TaskQueue queue )
+ public FileWorker( TaskQueue queue )
{
this.queue = queue;
}
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/FileWorker.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/WalkingDroid.java (from r699956, labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWalker.java)
URL: http://svn.apache.org/viewvc/labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/WalkingDroid.java?p2=labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/WalkingDroid.java&p1=labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWalker.java&r1=699956&r2=699965&rev=699965&view=diff
==============================================================================
--- labs/droids/branch/LABS-144/src/core/java/org/apache/droidsX/depot/filesystem/FileSystemWalker.java (original)
+++ labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/WalkingDroid.java Sun Sep 28 19:46:42 2008
@@ -14,23 +14,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droidsX.depot.filesystem;
+package org.apache.droids.walker;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
-import org.apache.droidsX.api.*;
-import org.apache.droidsX.impl.MultiThreadedTaskMaster;
-import org.apache.droidsX.impl.SimpleTaskQueue;
-public class FileSystemWalker implements Droid
+import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.droids.api.*;
+
+public class WalkingDroid implements Droid
{
private final TaskQueue queue;
private final TaskMaster taskMaster;
private Collection<File> initialFiles;
- public FileSystemWalker( TaskQueue queue, TaskMaster taskMaster )
+ public WalkingDroid( TaskQueue queue, TaskMaster taskMaster )
{
this.queue = queue;
this.taskMaster = taskMaster;
@@ -59,8 +60,8 @@
System.out.println( "FINISHED!!!" );
}
- public FileSystemWorker getNewWorker() {
- return new FileSystemWorker( queue );
+ public FileWorker getNewWorker() {
+ return new FileWorker( queue );
}
//------------------------------------------------------------------
@@ -76,7 +77,7 @@
Collection<File> files = new ArrayList<File>();
files.add( new File( args[0] ) ); //APACHE/droids/src" ) );
- FileSystemWalker simple = new FileSystemWalker( queue, taskMaster );
+ WalkingDroid simple = new WalkingDroid( queue, taskMaster );
simple.setInitialFiles( files );
simple.init();
simple.start(); // TODO? perhaps start internally calls init()?
Propchange: labs/droids/branch/LABS-144/src/robots/java/org/apache/droids/walker/WalkingDroid.java
------------------------------------------------------------------------------
svn:mergeinfo =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org