You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by bc...@apache.org on 2011/03/16 22:45:09 UTC
svn commit: r1082332 - in /incubator/droids/trunk:
droids-core/src/main/java/org/apache/droids/
droids-core/src/main/java/org/apache/droids/api/
droids-core/src/main/java/org/apache/droids/impl/
droids-core/src/main/java/org/apache/droids/robot/crawler...
Author: bchapuis
Date: Wed Mar 16 22:45:08 2011
New Revision: 1082332
URL: http://svn.apache.org/viewvc?rev=1082332&view=rev
Log:
DROIDS-56: Change the TaskQueue interface to java.util.Queue.
Added:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/TaskMasterImpl.java
Removed:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueue.java
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/AbstractDroid.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Droid.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/TestCrawlingWorker.java
incubator/droids/trunk/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml
incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml
incubator/droids/trunk/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java
incubator/droids/trunk/droids-wicket/src/main/resources/applicationContext.xml
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/AbstractDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/AbstractDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/AbstractDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/AbstractDroid.java Wed Mar 16 22:45:08 2011
@@ -16,23 +16,24 @@
*/
package org.apache.droids;
+import java.util.Queue;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Task;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Manage common tasks in standard Droids
*/
-public abstract class AbstractDroid<T extends Task> implements Droid<T>
+public abstract class AbstractDroid<T extends Task> implements Droid<T>
{
+
protected final Logger log = LoggerFactory.getLogger(AbstractDroid.class);
- protected final TaskQueue<T> queue;
+ protected final Queue<T> queue;
protected final TaskMaster<T> taskMaster;
-
- public AbstractDroid( TaskQueue<T> queue, TaskMaster<T> taskMaster )
+
+ public AbstractDroid(Queue<T> queue, TaskMaster<T> taskMaster)
{
this.queue = queue;
this.taskMaster = taskMaster;
@@ -41,16 +42,17 @@ public abstract class AbstractDroid<T ex
@Override
public void start()
{
- taskMaster.processAllTasks(queue, this);
+ taskMaster.start(queue, this);
}
-
- public TaskQueue<T> getQueue() {
+
+ public Queue<T> getQueue()
+ {
return queue;
}
@Override
- public TaskMaster<T> getTaskMaster() {
+ public TaskMaster<T> getTaskMaster()
+ {
return taskMaster;
}
}
-
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Droid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Droid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Droid.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Droid.java Wed Mar 16 22:45:08 2011
@@ -16,6 +16,7 @@
*/
package org.apache.droids.api;
+import java.util.Queue;
import org.apache.droids.exception.DroidsException;
/**
@@ -28,7 +29,8 @@ import org.apache.droids.exception.Droid
*
* @version 1.0
*/
-public interface Droid<T extends Task> {
+public interface Droid<T extends Task>
+{
/**
* Initialize the queue. Can have different implementation but the main groups
@@ -51,12 +53,19 @@ public interface Droid<T extends Task> {
* Invoke when the droid has completed
*/
void finished();
-
+
+ /**
+ * Return the tasks queue
+ *
+ * @return
+ */
+ public Queue<T> getQueue();
+
/**
* Ask the droid for a new worker
*/
Worker<T> getNewWorker();
-
+
/**
* Get the task master
*/
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java Wed Mar 16 22:45:08 2011
@@ -17,31 +17,58 @@
package org.apache.droids.api;
import java.util.Date;
+import java.util.Queue;
import java.util.concurrent.TimeUnit;
-
-
/**
* Responsible for running all the tasks
*/
-public interface TaskMaster<T extends Task> {
- public enum ExecutionState {
- INITALIZED,
- RUNNING,
- COMPLETE
+public interface TaskMaster<T extends Task>
+{
+
+ public enum ExecutionState
+ {
+ RUNNING, STOPPED, COMPLETED
};
-
- void processAllTasks( final TaskQueue<T> queue, final Droid<T> droid );
- Date getStartTime();
+ /**
+ * Start the process of running tasks.
+ * l
+ * @param droid
+ */
+ void start(final Queue<T> queue, final Droid<T> droid);
+
+ /**
+ * Stops the process of running tasks.
+ */
+ void stop();
+
+ /**
+ * Blocks until all tasks have completed execution.
+ *
+ * @param timeout
+ * @param unit
+ * @return
+ * @throws InterruptedException
+ */
+ boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException;
- Date getFinishedWorking();
+ void setExceptionHandler(TaskExceptionHandler taskExceptionHandler);
+ void setDelayTimer(DelayTimer simpleDelayTimer);
+
+ /**
+ * Returns the current state.
+ *
+ * @return
+ */
ExecutionState getExecutionState();
+ Date getStartTime();
+
+ Date getFinishedWorking();
+
long getCompletedTasks();
T getLastCompletedTask();
-
- boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException;
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java Wed Mar 16 22:45:08 2011
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.api;
-
-import java.util.Collection;
-
-import org.apache.droids.exception.InvalidTaskException;
-
-
-/**
- * A queue is the data structure where the different tasks are waiting for
- * service.
- * <p>
- * Droids use this object to determine the next task to process and the
- * overall workload.
- *
- * @version 1.0
- *
- */
-public interface TaskQueue<T extends Task> {
- /**
- * Return the task that is identified with the given id
- *
- * @param id
- * of the task we want
- * @return the task idenfied by the given id
- */
- T getTask(String id);
-
- /**
- * Do we have more task waiting for service
- *
- * @return true if we have. false otherwise
- */
- boolean hasNext();
-
- /**
- * Return the next task that is waiting for service
- *
- * @return next task in line.
- */
- T next();
-
- /**
- * Add a Task to the queue...
- *
- * @param task task to add to the Queue
- * @throws InvalidTaskException
- */
- void merge(T task) throws InvalidTaskException;
-
- /**
- * Merge a given list of tasks with the current queue. Whether or not we
- * eliminate double entries is up to the implementation
- *
- * @param tasks
- * the list of task that we want to add to the queue
- * @throws InvalidTaskException
- */
- void merge(Collection<? extends T> tasks) throws InvalidTaskException;
-
- /**
- * How many task do we have <em>left</em> in the queue.
- *
- * @return number of tasks do we have <em>left</em> in the queue.
- */
- int getSize();
-
- /**
- * Get the top 'size' elements in the queue
- */
- Collection<? extends T> peek( int size );
-}
\ No newline at end of file
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java Wed Mar 16 22:45:08 2011
@@ -16,36 +16,65 @@
*/
package org.apache.droids.impl;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedList;
import org.apache.droids.api.Task;
-import org.apache.droids.exception.InvalidTaskException;
/**
* Extend the task queue to ignore any tasks we have already seen
*/
-public class SimpleTaskQueueWithHistory<T extends Task> extends SimpleTaskQueue<T> {
+public class SimpleTaskQueueWithHistory<T extends Task> extends LinkedList<T>
+{
+
private final java.util.Set<String> previous;
/**
* Simple queue constructor.
*/
- public SimpleTaskQueueWithHistory() {
+ public SimpleTaskQueueWithHistory()
+ {
super();
- previous = Collections.synchronizedSet( new HashSet<String>() );
+ previous = Collections.synchronizedSet(new HashSet<String>());
+ }
+
+ @Override
+ public boolean offer(T e)
+ {
+ if (previous.add(e.getId())) {
+ return super.offer(e);
+ } else {
+ return false;
+ }
}
@Override
- public void merge(T task) throws InvalidTaskException {
- // only add it if we have not seen the taskId before...
- if( previous.add( task.getId() ) ) {
- super.merge( task );
+ public boolean add(T e)
+ {
+ if (previous.add(e.getId())) {
+ return super.offer(e);
+ } else {
+ return false;
}
}
-
+
@Override
- public void clear()
+ public boolean addAll(Collection<? extends T> c)
+ {
+ HashSet<String> ids = new HashSet<String>();
+ for (T e : c) {
+ ids.add(e.getId());
+ }
+ if (previous.addAll(ids)) {
+ return super.addAll(c);
+ } else {
+ return false;
+ }
+ }
+
+ public void clearHistory()
{
previous.clear();
}
Added: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/TaskMasterImpl.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/TaskMasterImpl.java?rev=1082332&view=auto
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/TaskMasterImpl.java (added)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/TaskMasterImpl.java Wed Mar 16 22:45:08 2011
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.impl;
+
+import java.util.Date;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.droids.AbstractDroid;
+import org.apache.droids.api.DelayTimer;
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Task;
+import org.apache.droids.api.TaskExceptionHandler;
+import org.apache.droids.api.TaskExceptionResult;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskMaster.ExecutionState;
+import org.apache.droids.api.WorkMonitor;
+import org.apache.droids.api.Worker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Responsible for running all the tasks
+ */
+public class TaskMasterImpl<T extends Task> implements TaskMaster<T>
+{
+
+ protected final Logger log = LoggerFactory.getLogger(AbstractDroid.class);
+ private final long TICKLE_TIME = 250L;
+ /**
+ * The execution state
+ */
+ protected volatile ExecutionState state = ExecutionState.STOPPED;
+ /**
+ * The delay timer
+ */
+ protected DelayTimer delayTimer;
+ /**
+ * The start time
+ */
+ protected Date startTime;
+ /**
+ * The end time
+ */
+ protected Date endTime;
+ /**
+ * The last completed task
+ */
+ protected T lastCompletedTask;
+ /**
+ * The completed task counter
+ */
+ protected AtomicLong completedTasks = new AtomicLong();
+ /**
+ * The monitor that that records the processing of tasks
+ */
+ protected WorkMonitor<T> monitor;
+ /**
+ * The task exception handler
+ */
+ protected TaskExceptionHandler exceptionHandler;
+
+ /*
+ * The pool size
+ */
+ private int poolSize = 1;
+ /**
+ * The pool
+ */
+ private TaskExecutorPool pool;
+
+ @Override
+ public void start(Queue<T> queue, Droid<T> droid)
+ {
+ if (log.isInfoEnabled()) {
+ log.info("Start the executor service.");
+ }
+
+ state = ExecutionState.RUNNING;
+
+ if (pool == null) {
+ this.pool = new TaskExecutorPool();
+ }
+
+ for (int i = 0; i < poolSize; i++) {
+ pool.execute(new TaskExecutor(droid));
+ }
+ }
+
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public void stop()
+ {
+ // debug
+ if (log.isInfoEnabled()) {
+ log.info("Stop the executor service.");
+ }
+
+ state = ExecutionState.STOPPED;
+
+ // Disable new tasks from being submitted
+ pool.shutdown();
+
+ // Wait a while for existing tasks to terminate
+ try {
+ if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
+
+ // Cancel currently executing tasks
+ pool.shutdownNow();
+
+ // Wait a while for to respond to being canceled
+ if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
+ if (log.isInfoEnabled()) {
+ log.info("Scheduler did not stop.");
+ }
+ }
+ }
+ } catch (InterruptedException ex) {
+
+ if (log.isInfoEnabled()) {
+ log.info("Force scheduler to stop.");
+ }
+
+ // (Re-)Cancel if current thread also interrupted
+ pool.shutdownNow();
+
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+
+ // debug
+ if (log.isInfoEnabled()) {
+ log.info("Scheduler stopped.");
+ }
+
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ return pool.awaitTermination(timeout, unit);
+ }
+
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public ExecutionState getExecutionState()
+ {
+ return state;
+ }
+
+ /**
+ * @return
+ * @inheritDoc
+ */
+ public WorkMonitor<T> getMonitor()
+ {
+ return monitor;
+ }
+
+ /**
+ * @param monitor
+ * @inheritDoc
+ */
+ public void setMonitor(WorkMonitor<T> monitor)
+ {
+ if (state == ExecutionState.RUNNING) {
+ throw new IllegalStateException("The TaskMaster must be stopped to set a Monitor.");
+ }
+ this.monitor = monitor;
+ }
+
+ @Override
+ public void setExceptionHandler(TaskExceptionHandler exceptionHandler)
+ {
+ this.exceptionHandler = exceptionHandler;
+ }
+
+ @Override
+ public void setDelayTimer(DelayTimer delayTimer)
+ {
+ this.delayTimer = delayTimer;
+ }
+
+ @Override
+ public Date getFinishedWorking()
+ {
+ return endTime;
+ }
+
+ @Override
+ public T getLastCompletedTask()
+ {
+ return lastCompletedTask;
+ }
+
+ @Override
+ public long getCompletedTasks()
+ {
+ return completedTasks.get();
+ }
+
+ @Override
+ public Date getStartTime()
+ {
+ return startTime;
+ }
+
+ /**
+ * Sets the pool size
+ *
+ * @return
+ */
+ public int getPoolSize()
+ {
+ return poolSize;
+ }
+
+ /**
+ * Returns the size of the pool
+ *
+ * @param poolSize
+ */
+ public void setPoolSize(int poolSize)
+ {
+ pool.setCorePoolSize(this.poolSize = poolSize);
+ }
+
+ private class TaskExecutorPool extends ThreadPoolExecutor
+ {
+
+ private static final long KEEP_ALIVE = 50000L;
+
+ public TaskExecutorPool()
+ {
+ super(poolSize, poolSize, KEEP_ALIVE, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
+ this.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
+ }
+
+ @Override
+ protected void afterExecute(Runnable r, Throwable thrwbl)
+ {
+ super.afterExecute(r, thrwbl);
+
+ // try to reexecute the task runner while
+ // the task queue is not empty and while the pool
+ // is still completing the execution of tasks.
+ TaskExecutor taskExecutor = (TaskExecutor) r;
+ while (taskExecutor.getQueue().size() > 0 || getQueue().size() > 0) {
+ if (taskExecutor.getQueue().size() > 0) {
+ execute(r);
+ return;
+ }
+ try {
+ Thread.sleep(TICKLE_TIME);
+ } catch (InterruptedException e) {
+ log.error("", e);
+ }
+ }
+
+ taskExecutor.getDroid().finished();
+ state = ExecutionState.COMPLETED;
+ shutdownNow();
+
+ }
+ }
+
+ private class TaskExecutor implements Runnable
+ {
+
+ private final Droid<T> droid;
+ private final Queue<T> queue;
+ private final Worker<T> worker;
+
+ public TaskExecutor(Droid<T> droid)
+ {
+ this.droid = droid;
+ this.queue = droid.getQueue();
+ this.worker = droid.getNewWorker();
+ }
+
+ public Droid<T> getDroid()
+ {
+ return droid;
+ }
+
+ public Queue<T> getQueue()
+ {
+ return queue;
+ }
+
+ public Worker getWorker()
+ {
+ return worker;
+ }
+
+ @Override
+ public void run()
+ {
+ // poll the last task
+ T task = queue.poll();
+
+ // execute the task
+ if (task != null) {
+ try {
+ // monitor the execution of the task
+ if (monitor != null) {
+ monitor.beforeExecute(task, worker);
+ }
+
+ // debug
+ if (log.isDebugEnabled()) {
+ log.debug("Worker [" + worker + "] execute task [" + task + "].");
+ }
+
+ // execute the task
+ worker.execute(task);
+
+ // debug
+ if (log.isDebugEnabled()) {
+ log.debug("Worker [" + worker + "] executed task [" + task + "] with success.");
+ }
+
+ // monitor the execution of the task
+ if (monitor != null) {
+ monitor.afterExecute(task, worker, null);
+ }
+
+ // set the monitored variables
+ completedTasks.incrementAndGet();
+ lastCompletedTask = task;
+
+ } catch (Exception ex) {
+ // debug
+ if (log.isDebugEnabled()) {
+ log.debug("Worker [" + worker + "] executed task [" + task + "] without success.");
+ }
+
+ // debug
+ if (log.isErrorEnabled()) {
+ log.error("", ex);
+ }
+
+ // monitor the exception
+ if (monitor != null) {
+ monitor.afterExecute(task, worker, ex);
+ }
+
+ // handler the exception
+ if (ex != null) {
+ TaskExceptionResult result = exceptionHandler.handleException(ex);
+
+ // stop the execution in case of a fatal exception
+ if (TaskExceptionResult.FATAL.equals(result)) {
+ state = ExecutionState.STOPPED;
+ }
+
+ droid.finished();
+ pool.shutdownNow();
+
+ }
+ }
+ }
+ }
+ }
+}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java Wed Mar 16 22:45:08 2011
@@ -21,85 +21,92 @@ import java.net.URISyntaxException;
import java.util.Collection;
import com.google.common.base.Preconditions;
+import java.util.Queue;
import org.apache.droids.AbstractDroid;
import org.apache.droids.LinkTask;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.exception.InvalidTaskException;
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
-public abstract class CrawlingDroid extends AbstractDroid<Link>
+public abstract class CrawlingDroid extends AbstractDroid<Link>
{
- private Collection<String> initialLocations;
+ private Collection<String> initialLocations;
ProtocolFactory protocolFactory;
ParserFactory parserFactory;
URLFiltersFactory filtersFactory;
-
- public CrawlingDroid( TaskQueue<Link> queue, TaskMaster<Link> taskMaster )
+
+ public CrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster)
{
- super( queue, taskMaster );
+ super(queue, taskMaster);
}
- public void setInitialLocations(Collection<String> initialLocations) {
+ public void setInitialLocations(Collection<String> initialLocations)
+ {
this.initialLocations = initialLocations;
}
@Override
- public void init() throws InvalidTaskException {
- Preconditions.checkState( initialLocations != null, "WebCrawlerDroid requires at least one starting file" );
- Preconditions.checkState( !initialLocations.isEmpty(), "WebCrawlerDroid requires at least one starting file" );
- for( String location : initialLocations ) {
+ public void init() throws InvalidTaskException
+ {
+ Preconditions.checkState(initialLocations != null, "WebCrawlerDroid requires at least one starting file");
+ Preconditions.checkState(!initialLocations.isEmpty(), "WebCrawlerDroid requires at least one starting file");
+ for (String location : initialLocations) {
URI uri;
try {
uri = new URI(location);
} catch (URISyntaxException ex) {
throw new InvalidTaskException("Invalid lication: " + location);
}
- queue.merge( new LinkTask( null, uri, 0 ) );
+ queue.offer(new LinkTask(null, uri, 0));
}
}
-
+
public void start()
{
- taskMaster.processAllTasks(queue, this);
+ taskMaster.start(queue, this);
}
@Override
public void finished()
{
- log.info( "FINISHED!!!" );
+ log.info("FINISHED!!!");
}
public abstract Worker<Link> getNewWorker();
- public ProtocolFactory getProtocolFactory() {
+ public ProtocolFactory getProtocolFactory()
+ {
return protocolFactory;
}
- public void setProtocolFactory(ProtocolFactory protocolFactory) {
+ public void setProtocolFactory(ProtocolFactory protocolFactory)
+ {
this.protocolFactory = protocolFactory;
}
- public ParserFactory getParserFactory() {
+ public ParserFactory getParserFactory()
+ {
return parserFactory;
}
- public void setParserFactory(ParserFactory parserFactory) {
+ public void setParserFactory(ParserFactory parserFactory)
+ {
this.parserFactory = parserFactory;
}
- public URLFiltersFactory getFiltersFactory() {
+ public URLFiltersFactory getFiltersFactory()
+ {
return filtersFactory;
}
- public void setFiltersFactory(URLFiltersFactory filtersFactory) {
+ public void setFiltersFactory(URLFiltersFactory filtersFactory)
+ {
this.filtersFactory = filtersFactory;
}
}
-
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java Wed Mar 16 22:45:08 2011
@@ -88,7 +88,7 @@ public class CrawlingWorker implements W
Parse parse = parser.parse(entity, link);
if( parse.getOutlinks() != null ) {
Collection<Link> outlinks = getFilteredOutlinks( parse );
- droid.getQueue().merge( outlinks );
+ droid.getQueue().addAll( outlinks );
}
entity.setParse(parse);
handle(entity, link);
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java Wed Mar 16 22:45:08 2011
@@ -18,7 +18,7 @@ package org.apache.droids.robot.walker;
import java.io.File;
-import org.apache.droids.api.TaskQueue;
+import java.util.Queue;
import org.apache.droids.api.Worker;
import org.apache.droids.exception.InvalidTaskException;
import org.slf4j.Logger;
@@ -26,11 +26,11 @@ import org.slf4j.LoggerFactory;
public class FileWorker implements Worker<FileTask>
{
+
private final Logger log = LoggerFactory.getLogger(FileWorker.class);
+ final Queue<FileTask> queue;
- final TaskQueue<FileTask> queue;
-
- public FileWorker( TaskQueue<FileTask> queue )
+ public FileWorker(Queue<FileTask> queue)
{
this.queue = queue;
}
@@ -39,17 +39,16 @@ public class FileWorker implements Worke
public void execute(FileTask task) throws InvalidTaskException
{
File file = task.getFile();
- if( file.isDirectory() ) {
+ if (file.isDirectory()) {
File[] files = file.listFiles();
- if( files != null ) {
- for( File f : files ) {
- queue.merge( new FileTask( f, task.getDepth()+1 ) );
+ if (files != null) {
+ for (File f : files) {
+ queue.add(new FileTask(f, task.getDepth() + 1));
}
}
- }
- else {
+ } else {
if (log.isInfoEnabled()) {
- log.info( "FILE: "+file.getAbsolutePath() );
+ log.info("FILE: " + file.getAbsolutePath());
}
}
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java Wed Mar 16 22:45:08 2011
@@ -22,65 +22,68 @@ import java.util.Collection;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
+import java.util.LinkedList;
+import java.util.Queue;
import org.apache.droids.AbstractDroid;
import org.apache.droids.exception.InvalidTaskException;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
import org.apache.droids.api.*;
+import org.apache.droids.impl.TaskMasterImpl;
public class SimpleWalkingDroid extends AbstractDroid<FileTask> implements WalkingDroid
{
+
private Collection<File> initialFiles;
-
- public SimpleWalkingDroid( TaskQueue<FileTask> queue, TaskMaster<FileTask> taskMaster )
+
+ public SimpleWalkingDroid(Queue<FileTask> queue, TaskMaster<FileTask> taskMaster)
{
- super( queue, taskMaster );
+ super(queue, taskMaster);
}
@Override
- public void setInitialFiles(Collection<File> initialFiles) {
+ public void setInitialFiles(Collection<File> initialFiles)
+ {
this.initialFiles = initialFiles;
}
@Override
- public void init() throws InvalidTaskException {
- Preconditions.checkState(initialFiles != null, "FileSystemWalker requires at least one starting file" );
- Preconditions.checkState( !initialFiles.isEmpty(), "FileSystemWalker requires at least one starting file" );
- for( File file : initialFiles ) {
- queue.merge( new FileTask( file, 0 ) );
+ public void init() throws InvalidTaskException
+ {
+ Preconditions.checkState(initialFiles != null, "FileSystemWalker requires at least one starting file");
+ Preconditions.checkState(!initialFiles.isEmpty(), "FileSystemWalker requires at least one starting file");
+ for (File file : initialFiles) {
+ queue.add(new FileTask(file, 0));
}
}
@Override
public void finished()
{
- log.info( "FINISHED!!!" );
+ log.info("FINISHED!!!");
}
@Override
- public FileWorker getNewWorker() {
- return new FileWorker( queue );
+ public FileWorker getNewWorker()
+ {
+ return new FileWorker(queue);
}
//------------------------------------------------------------------
//------------------------------------------------------------------
-
- public static void main( String[] args ) throws Exception
+ public static void main(String[] args) throws Exception
{
- MultiThreadedTaskMaster<FileTask> taskMaster = new MultiThreadedTaskMaster<FileTask>();
- taskMaster.setMaxThreads( 3 );
-
- TaskQueue<FileTask> queue = new SimpleTaskQueue<FileTask>();
-
+ TaskMasterImpl<FileTask> taskMaster = new TaskMasterImpl<FileTask>();
+ taskMaster.setPoolSize(3);
+
+ Queue<FileTask> queue = new LinkedList<FileTask>();
+
Collection<File> files = new ArrayList<File>();
- files.add( new File( args[0] ) ); //APACHE/droids/src" ) );
-
- SimpleWalkingDroid simple = new SimpleWalkingDroid( queue, taskMaster );
- simple.setInitialFiles( files );
+ files.add(new File(args[0])); //APACHE/droids/src" ) );
+
+ SimpleWalkingDroid simple = new SimpleWalkingDroid(queue, taskMaster);
+ simple.setInitialFiles(files);
simple.init();
simple.start(); // TODO? perhaps start internally calls init()?
simple.getTaskMaster().awaitTermination(0, TimeUnit.SECONDS);
}
}
-
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java Wed Mar 16 22:45:08 2011
@@ -16,156 +16,161 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.droids;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-
-import org.apache.droids.api.Droid;
-import org.apache.droids.api.Handler;
-import org.apache.droids.api.Link;
-import org.apache.droids.api.URLFilter;
-import org.apache.droids.delay.SimpleDelayTimer;
+package org.apache.droids;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Queue;
+
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Handler;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.URLFilter;
+import org.apache.droids.delay.SimpleDelayTimer;
import org.apache.droids.handle.SaveHandler;
-import org.apache.droids.helper.factories.HandlerFactory;
-import org.apache.droids.helper.factories.ParserFactory;
-import org.apache.droids.helper.factories.ProtocolFactory;
-import org.apache.droids.helper.factories.URLFiltersFactory;
-import org.apache.droids.impl.DefaultTaskExceptionHandler;
-import org.apache.droids.impl.ExceptionCrawlingDroid;
-import org.apache.droids.impl.ReportCrawlingDroid;
-import org.apache.droids.impl.SaveCrawlingDroid;
-import org.apache.droids.impl.SequentialTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
-import org.apache.droids.impl.SimpleTaskQueueWithHistory;
-import org.apache.droids.parse.html.HtmlParser;
-import org.apache.droids.protocol.http.HttpProtocol;
-import org.apache.droids.robot.crawler.CrawlingDroid;
-
-public class DroidsFactory
-{
-
- public static ParserFactory createDefaultParserFactory() {
- ParserFactory parserFactory = new ParserFactory();
- HtmlParser htmlParser = new HtmlParser();
- htmlParser.setElements(new HashMap<String, String>());
- htmlParser.getElements().put("a", "href");
- htmlParser.getElements().put("link", "href");
- htmlParser.getElements().put("img", "src");
- htmlParser.getElements().put("script", "src");
- parserFactory.setMap(new HashMap<String, Object>());
- parserFactory.getMap().put("text/html", htmlParser);
- return parserFactory;
- }
-
- public static ProtocolFactory createDefaultProtocolFactory() {
- ProtocolFactory protocolFactory = new ProtocolFactory();
- HttpProtocol httpProtocol = new HttpProtocol();
- httpProtocol.setForceAllow(true);
-
- protocolFactory.setMap(new HashMap<String, Object>());
- protocolFactory.getMap().put("http", httpProtocol);
- return protocolFactory;
- }
-
- public static URLFiltersFactory createDefaultURLFiltersFactory() {
- URLFiltersFactory filtersFactory = new URLFiltersFactory();
- URLFilter defaultURLFilter = new URLFilter() {
-
- public String filter(String urlString) {
- return urlString;
- }
-
- };
- filtersFactory.setMap(new HashMap<String, Object>());
- filtersFactory.getMap().put("default", defaultURLFilter);
- return filtersFactory;
- }
-
- public static HandlerFactory createDefaultHandlerFactory(Handler defaultHandler) {
- HandlerFactory handlerFactory = new HandlerFactory();
- handlerFactory.setMap(new HashMap<String, Object>());
- handlerFactory.getMap().put("default", defaultHandler);
- return handlerFactory;
- }
-
- public static Droid<Link> createSimpleSaveCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SimpleTaskQueueWithHistory<Link> simpleQueue = new SimpleTaskQueueWithHistory<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
+import org.apache.droids.helper.factories.HandlerFactory;
+import org.apache.droids.helper.factories.ParserFactory;
+import org.apache.droids.helper.factories.ProtocolFactory;
+import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.impl.DefaultTaskExceptionHandler;
+import org.apache.droids.impl.TaskMasterImpl;
+import org.apache.droids.impl.ExceptionCrawlingDroid;
+import org.apache.droids.impl.ReportCrawlingDroid;
+import org.apache.droids.impl.SaveCrawlingDroid;
+import org.apache.droids.impl.SimpleTaskQueueWithHistory;
+import org.apache.droids.parse.html.HtmlParser;
+import org.apache.droids.protocol.http.HttpProtocol;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+
+public class DroidsFactory
+{
+
+ public static ParserFactory createDefaultParserFactory() {
+ ParserFactory parserFactory = new ParserFactory();
+ HtmlParser htmlParser = new HtmlParser();
+ htmlParser.setElements(new HashMap<String, String>());
+ htmlParser.getElements().put("a", "href");
+ htmlParser.getElements().put("link", "href");
+ htmlParser.getElements().put("img", "src");
+ htmlParser.getElements().put("script", "src");
+ parserFactory.setMap(new HashMap<String, Object>());
+ parserFactory.getMap().put("text/html", htmlParser);
+ return parserFactory;
+ }
+
+ public static ProtocolFactory createDefaultProtocolFactory() {
+ ProtocolFactory protocolFactory = new ProtocolFactory();
+ HttpProtocol httpProtocol = new HttpProtocol();
+ httpProtocol.setForceAllow(true);
+
+ protocolFactory.setMap(new HashMap<String, Object>());
+ protocolFactory.getMap().put("http", httpProtocol);
+ return protocolFactory;
+ }
+
+ public static URLFiltersFactory createDefaultURLFiltersFactory() {
+ URLFiltersFactory filtersFactory = new URLFiltersFactory();
+ URLFilter defaultURLFilter = new URLFilter() {
+
+ public String filter(String urlString) {
+ return urlString;
+ }
+
+ };
+ filtersFactory.setMap(new HashMap<String, Object>());
+ filtersFactory.getMap().put("default", defaultURLFilter);
+ return filtersFactory;
+ }
+
+ public static HandlerFactory createDefaultHandlerFactory(Handler defaultHandler) {
+ HandlerFactory handlerFactory = new HandlerFactory();
+ handlerFactory.setMap(new HashMap<String, Object>());
+ handlerFactory.getMap().put("default", defaultHandler);
+ return handlerFactory;
+ }
+
+ public static Droid<Link> createSimpleSaveCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ SimpleTaskQueueWithHistory<Link> simpleQueue = new SimpleTaskQueueWithHistory<Link>();
+
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster, new SaveHandler() );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
- public static Droid<Link> createSimpleReportCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- SimpleTaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-
- CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
- public static Droid<Link> createSimpleExceptionCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SimpleTaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
-}
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleReportCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+// SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
+// MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
+ Queue<Link> queue = new LinkedList<Link>();
+
+ CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleExceptionCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ Queue<Link> queue = new LinkedList<Link>();
+
+// SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
+// MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
+ CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+}
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java Wed Mar 16 22:45:08 2011
@@ -22,104 +22,103 @@ import java.util.Collection;
import java.util.LinkedHashMap;
import com.google.common.base.Preconditions;
+import java.util.LinkedList;
+import java.util.Queue;
import org.apache.droids.AbstractDroid;
import org.apache.droids.api.*;
-import org.apache.droids.exception.InvalidTaskException;
import org.apache.droids.robot.walker.FileTask;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.droids.impl.TaskMasterImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
public class FileRenameDroid extends AbstractDroid<FileTask>
{
+
private static final Logger log = LoggerFactory.getLogger(FileRenameDroid.class);
-
private Collection<File> initialFiles;
-
- public FileRenameDroid( TaskQueue<FileTask> queue, TaskMaster<FileTask> taskMaster )
+
+ public FileRenameDroid(Queue<FileTask> queue, TaskMaster<FileTask> taskMaster)
{
- super( queue, taskMaster );
+ super(queue, taskMaster);
}
- public void setInitialFiles(Collection<File> initialFiles) {
+ public void setInitialFiles(Collection<File> initialFiles)
+ {
this.initialFiles = initialFiles;
}
-
- public void init() {
+
+ public void init()
+ {
Preconditions.checkNotNull(initialFiles);
- Preconditions.checkState( !initialFiles.isEmpty() );
- try {
- for( File file : initialFiles ) {
- queue.merge( new FileTask( file, 0 ) );
- }
- }
- catch (InvalidTaskException e) {
- e.printStackTrace();
+ Preconditions.checkState(!initialFiles.isEmpty());
+ for (File file : initialFiles) {
+ queue.add(new FileTask(file, 0));
}
}
-
-
public LinkedHashMap<String, String> cleaner = null;
- public LinkedHashMap<String, String> getCleaner() {
+ public LinkedHashMap<String, String> getCleaner()
+ {
if (null == cleaner) {
populateCleaner();
}
return cleaner;
}
- public void setCleaner(LinkedHashMap<String, String> cleaner) {
+ public void setCleaner(LinkedHashMap<String, String> cleaner)
+ {
this.cleaner = cleaner;
}
- private void populateCleaner() {
+ private void populateCleaner()
+ {
cleaner = new LinkedHashMap<String, String>();
cleaner.put(" ", ".");
cleaner.put(".-.", ".");
cleaner.put(",", "");
}
-
+
public void finished()
{
- System.out.println( "FINISHED!!!" );
+ System.out.println("FINISHED!!!");
}
- public RenameWorker getNewWorker() {
+ public RenameWorker getNewWorker()
+ {
return new RenameWorker();
}
+ public class RenameWorker implements Worker<FileTask>
+ {
- public class RenameWorker implements Worker<FileTask> {
-
String replace;
- public void execute(FileTask task) {
+ public void execute(FileTask task)
+ {
- for (String pattern : getCleaner().keySet() ) {
+ for (String pattern : getCleaner().keySet()) {
replace = getCleaner().get(pattern);
-
- cleanFileName(task.getFile(), pattern, replace);
+
+ cleanFileName(task.getFile(), pattern, replace);
}
}
-
- private void cleanFileName(File file, String pattern, String replace) {
+
+ private void cleanFileName(File file, String pattern, String replace)
+ {
log.debug("Processing: " + file.getName());
log.debug("finding pattern: " + pattern);
log.debug("replacing it with: " + replace);
String fileName = file.getName();
if (fileName.contains(pattern) || !fileName.toLowerCase().equals(fileName)) {
log.debug("need to process this file: " + fileName + " in "
- + file.getAbsolutePath());
- File replacement = new File(fileName.substring(0, file.getAbsolutePath()
- .indexOf(fileName))
- + fileName.replaceAll(pattern, replace).toLowerCase());
+ + file.getAbsolutePath());
+ File replacement = new File(fileName.substring(0, file.getAbsolutePath().indexOf(fileName))
+ + fileName.replaceAll(pattern, replace).toLowerCase());
log.debug("Renaming to: " + replacement.getName() + " in "
- + replacement.getAbsolutePath());
+ + replacement.getAbsolutePath());
- log.info( "TODO! actually do the rename!" );
+ log.info("TODO! actually do the rename!");
// TODO -- actually do the rename...file.renameTo(replacement);
}
}
@@ -127,21 +126,19 @@ public class FileRenameDroid extends Abs
//------------------------------------------------------------------
//------------------------------------------------------------------
-
- public static void main( String[] args )
+ public static void main(String[] args)
{
- MultiThreadedTaskMaster<FileTask> taskMaster = new MultiThreadedTaskMaster<FileTask>();
- taskMaster.setMaxThreads( 3 );
-
- TaskQueue<FileTask> queue = new SimpleTaskQueue<FileTask>();
-
+ TaskMasterImpl<FileTask> taskMaster = new TaskMasterImpl<FileTask>();
+ taskMaster.setPoolSize(3);
+
+ Queue<FileTask> queue = new LinkedList<FileTask>();
+
Collection<File> files = new ArrayList<File>();
- files.add( new File( args[0] ) );
-
- FileRenameDroid simple = new FileRenameDroid( queue, taskMaster );
- simple.setInitialFiles( files );
+ files.add(new File(args[0]));
+
+ FileRenameDroid simple = new FileRenameDroid(queue, taskMaster);
+ simple.setInitialFiles(files);
simple.init();
simple.start(); // TODO? perhaps start internally calls init()?
}
}
-
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java Wed Mar 16 22:45:08 2011
@@ -19,6 +19,8 @@ package org.apache.droids.examples;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Queue;
import java.util.concurrent.TimeUnit;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -31,8 +33,7 @@ import org.apache.droids.helper.factorie
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
import org.apache.droids.impl.DefaultTaskExceptionHandler;
-import org.apache.droids.impl.SequentialTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.droids.impl.TaskMasterImpl;
import org.apache.droids.impl.SysoutCrawlingDroid;
import org.apache.droids.net.RegexURLFilter;
import org.apache.droids.parse.html.HtmlParser;
@@ -124,11 +125,9 @@ public class SimpleRuntime {
SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
simpleDelayTimer.setDelayMillis(100);
- SimpleTaskQueue<Link> simpleQueue = new SimpleTaskQueue<Link>();
- // simpleQueue.setMaxDepth(3);
- // simpleQueue.setMaxSize(-1);
+ Queue<Link> simpleQueue = new LinkedList<Link>();
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
taskMaster.setDelayTimer( simpleDelayTimer );
taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java Wed Mar 16 22:45:08 2011
@@ -18,11 +18,11 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.examples.handler.ExceptionReportHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -30,7 +30,7 @@ import org.apache.droids.robot.crawler.C
public class ExceptionCrawlingDroid extends CrawlingDroid {
- public ExceptionCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
+ public ExceptionCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
super(queue, taskMaster);
}
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java Wed Mar 16 22:45:08 2011
@@ -18,10 +18,10 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.examples.handler.ReportHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -29,7 +29,7 @@ import org.apache.droids.robot.crawler.C
public class ReportCrawlingDroid extends CrawlingDroid {
- public ReportCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
+ public ReportCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
super(queue, taskMaster);
}
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java Wed Mar 16 22:45:08 2011
@@ -18,11 +18,11 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.robot.crawler.CrawlingDroid;
import org.apache.droids.robot.crawler.CrawlingWorker;
@@ -30,7 +30,7 @@ import org.apache.droids.robot.crawler.C
public class SaveCrawlingDroid extends CrawlingDroid {
private final Handler defaultHandler;
- public SaveCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster, final Handler defaultHandlerForWorkerCreation ) {
+ public SaveCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster, final Handler defaultHandlerForWorkerCreation ) {
super(queue, taskMaster);
assert( defaultHandlerForWorkerCreation != null );
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java Wed Mar 16 22:45:08 2011
@@ -18,10 +18,10 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.handle.SysoutHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -29,7 +29,7 @@ import org.apache.droids.robot.crawler.C
public class SysoutCrawlingDroid extends CrawlingDroid {
- public SysoutCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
+ public SysoutCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
super(queue, taskMaster);
}
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java Wed Mar 16 22:45:08 2011
@@ -58,11 +58,12 @@ public class TestSimpleDroid
String targetURI = baseURI + "/start_html";
- Droid<Link> droid = DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
+ Droid<Link> droid = DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
droid.init();
droid.start();
- droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
+
+ while (!droid.getTaskMaster().awaitTermination(250L, TimeUnit.MILLISECONDS));
Assert.assertFalse(ReportHandler.getReport().isEmpty());
Assert.assertEquals(5, ReportHandler.getReport().size());
@@ -87,7 +88,7 @@ public class TestSimpleDroid
Droid<Link> droid = DroidsFactory.createSimpleExceptionCrawlingDroid(
targetURI);
- SequentialTaskMaster<Link> taskMaster = (SequentialTaskMaster<Link>) droid.getTaskMaster();
+ TaskMasterImpl<Link> taskMaster = (TaskMasterImpl<Link>) droid.getTaskMaster();
taskMaster.setExceptionHandler(new TaskExceptionHandler() {
public TaskExceptionResult handleException(Exception ex) {
@@ -96,12 +97,12 @@ public class TestSimpleDroid
}
return TaskExceptionResult.WARN;
}
-
+
});
droid.init();
droid.start();
- droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
+ while (!droid.getTaskMaster().awaitTermination(250L, TimeUnit.MILLISECONDS));
Assert.assertFalse(ReportHandler.getReport().isEmpty());
Assert.assertEquals(4, ReportHandler.getReport().size());
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/TestCrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/TestCrawlingWorker.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/TestCrawlingWorker.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/TestCrawlingWorker.java Wed Mar 16 22:45:08 2011
@@ -4,16 +4,16 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.LinkTask;
import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Link;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.exception.DroidsException;
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
-import org.apache.droids.impl.SimpleTaskQueue;
import org.apache.droids.impl.SysoutCrawlingDroid;
import org.apache.droids.parse.html.HtmlParser;
import org.junit.After;
@@ -29,12 +29,12 @@ public class TestCrawlingWorker {
@Before
public void initialize() {
- final TaskQueue<Link> queue = new SimpleTaskQueue<Link>();
+ final Queue<Link> queue = new LinkedList<Link>();
final CrawlingDroid droid = createDroid(queue);
instance = (CrawlingWorker) droid.getNewWorker();
}
- private final CrawlingDroid createDroid(final TaskQueue<Link> queue) {
+ private final CrawlingDroid createDroid(final Queue<Link> queue) {
final CrawlingDroid droid = new SysoutCrawlingDroid(queue, null);
final ProtocolFactory protocolFactory = DroidsFactory.createDefaultProtocolFactory();
Modified: incubator/droids/trunk/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml (original)
+++ incubator/droids/trunk/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml Wed Mar 16 22:45:08 2011
@@ -44,7 +44,7 @@
</bean>
<bean name="taskMaster"
- class="org.apache.droids.impl.SequentialTaskMaster">
+ class="org.apache.droids.impl.TaskMasterImpl">
<property name="exceptionHandler" ref="taskExceptionHandler" />
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
<!--<property name="maxThreads" value="${droids.maxThreads}"/>-->
@@ -53,17 +53,16 @@
<!-- Droids -->
<bean name="org.apache.droids.api.Droid/hello"
class="org.apache.droids.impl.SysoutCrawlingDroid">
+ <constructor-arg ref="java.util.LinkedList" />
<constructor-arg ref="taskMaster" />
- <constructor-arg ref="org.apache.droids.impl.SimpleTaskQueue" />
<property name="protocolFactory" ref="org.apache.droids.helper.factories.ProtocolFactory"/>
<property name="parserFactory" ref="org.apache.droids.helper.factories.ParserFactory"/>
<property name="filtersFactory" ref="org.apache.droids.helper.factories.URLFiltersFactory"/>
</bean>
<!-- Queue -->
- <bean id="org.apache.droids.impl.SimpleTaskQueue"
- class="org.apache.droids.impl.SimpleTaskQueue">
- <property name="maxSize" value="${droids.queue.maxSize}"/>
+ <bean id="java.util.LinkedList"
+ class="java.util.LinkedList">
</bean>
<!-- Protocol -->
<bean
Modified: incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java (original)
+++ incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java Wed Mar 16 22:45:08 2011
@@ -22,14 +22,10 @@ import java.util.concurrent.TimeUnit;
import junit.framework.Assert;
-import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Link;
-import org.apache.droids.api.TaskExceptionHandler;
-import org.apache.droids.api.TaskExceptionResult;
import org.apache.droids.examples.handler.ReportHandler;
import org.apache.droids.impl.ReportCrawlingDroid;
-import org.apache.droids.impl.SequentialTaskMaster;
import org.apache.droids.localserver.LocalHttpServer;
import org.apache.droids.localserver.ResourceHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
Modified: incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml (original)
+++ incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml Wed Mar 16 22:45:08 2011
@@ -44,7 +44,7 @@
</bean>
<bean name="taskMaster"
- class="org.apache.droids.impl.SequentialTaskMaster">
+ class="org.apache.droids.impl.TaskMasterImpl">
<property name="exceptionHandler" ref="taskExceptionHandler" />
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
<!--<property name="maxThreads" value="${droids.maxThreads}"/>-->
@@ -53,17 +53,17 @@
<!-- Droids -->
<bean name="org.apache.droids.api.Droid/report"
class="org.apache.droids.impl.ReportCrawlingDroid">
+ <constructor-arg ref="java.util.LinkedList" />
<constructor-arg ref="taskMaster" />
- <constructor-arg ref="org.apache.droids.impl.SimpleTaskQueue" />
+
<property name="protocolFactory" ref="org.apache.droids.helper.factories.ProtocolFactory"/>
<property name="parserFactory" ref="org.apache.droids.helper.factories.ParserFactory"/>
<property name="filtersFactory" ref="org.apache.droids.helper.factories.URLFiltersFactory"/>
</bean>
<!-- Queue -->
- <bean id="org.apache.droids.impl.SimpleTaskQueue"
- class="org.apache.droids.impl.SimpleTaskQueue">
- <property name="maxSize" value="${droids.queue.maxSize}"/>
+ <bean id="java.util.LinkedList"
+ class="java.util.LinkedList">
</bean>
<!-- Protocol -->
<bean
Modified: incubator/droids/trunk/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java (original)
+++ incubator/droids/trunk/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java Wed Mar 16 22:45:08 2011
@@ -17,17 +17,14 @@
package org.apache.droids.wicket.component;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
-import java.util.List;
+import java.util.Queue;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Task;
-import org.apache.droids.api.TaskQueue;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.TaskMasterImpl;
import org.apache.droids.monitor.SimpleWorkMonitor;
import org.apache.droids.monitor.WorkBean;
import org.apache.wicket.ajax.AbstractAjaxTimerBehavior;
@@ -54,10 +51,10 @@ public class DroidPanel extends Panel
Droid droid;
@SpringBean( name="taskQueue" )
- TaskQueue taskQueue;
+ Queue taskQueue;
@SpringBean( name="taskMaster" )
- MultiThreadedTaskMaster taskMaster;
+ TaskMasterImpl taskMaster;
@SpringBean( name="runningMonitor" )
SimpleWorkMonitor monitor;
@@ -173,18 +170,7 @@ public class DroidPanel extends Panel
@Override
protected Iterator<IModel<Task>> getItemModels()
{
- Collection<Task> top = taskQueue.peek( 5 );
-
- List<IModel<Task>> models = new ArrayList<IModel<Task>>( top.size() );
- for( final Task m : top ) {
- models.add( new AbstractReadOnlyModel<Task>() {
- @Override
- public Task getObject() {
- return m;
- }
- });
- }
- return models.iterator();
+ return taskQueue.iterator();
}
@Override
@@ -198,7 +184,7 @@ public class DroidPanel extends Panel
queueDIV.add( new Label( "more", new AbstractReadOnlyModel<String>() {
@Override
public String getObject() {
- return taskQueue.getSize() + " tasks.";
+ return taskQueue.size() + " tasks.";
}
}));
queueDIV.setOutputMarkupId( true );
Modified: incubator/droids/trunk/droids-wicket/src/main/resources/applicationContext.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-wicket/src/main/resources/applicationContext.xml?rev=1082332&r1=1082331&r2=1082332&view=diff
==============================================================================
--- incubator/droids/trunk/droids-wicket/src/main/resources/applicationContext.xml (original)
+++ incubator/droids/trunk/droids-wicket/src/main/resources/applicationContext.xml Wed Mar 16 22:45:08 2011
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
@@ -17,34 +17,34 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<!DOCTYPE beans PUBLIC "-//SPRING//DTD BEAN//EN" "http://www.springframework.org/dtd/spring-beans.dtd">
-
+<!DOCTYPE beans PUBLIC "-//SPRING//DTD BEAN//EN" "http://www.springframework.org/dtd/spring-beans.dtd">
+
<beans>
<!-- use system variables -->
<bean id="placeholderConfig"
class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="location"><value>classpath:application.properties</value></property>
<property name="systemPropertiesModeName"><value>SYSTEM_PROPERTIES_MODE_OVERRIDE</value></property>
- </bean>
-
- <!-- setup wicket application -->
- <bean id="wicketApplication" class="org.apache.droids.wicket.app.DroidsApplication" />
-
-
- <bean id="runningMonitor" class="org.apache.droids.monitor.SimpleWorkMonitor" />
+ </bean>
+
+ <!-- setup wicket application -->
+ <bean id="wicketApplication" class="org.apache.droids.wicket.app.DroidsApplication" />
+
+
+ <bean id="runningMonitor" class="org.apache.droids.monitor.SimpleWorkMonitor" />
<!-- DROIDS -->
<bean name="taskMaster"
- class="org.apache.droids.impl.MultiThreadedTaskMaster">
+ class="org.apache.droids.impl.TaskMasterImpl">
<!--
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
-->
- <property name="maxThreads" value="1"/>
+ <property name="maxThreads" value="1"/>
<property name="monitor" ref="runningMonitor"/>
</bean>
<bean name="taskQueue"
- class="org.apache.droids.impl.SimpleTaskQueue">
+ class="java.util.concurrent.ConcurrentLinkedQueue">
<!-- class="org.apache.droids.impl.SimpleTaskQueue" -->
</bean>
@@ -54,4 +54,4 @@
<constructor-arg ref="taskMaster" />
</bean>
-</beans>
+</beans>