You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by bc...@apache.org on 2011/02/02 22:29:29 UTC
svn commit: r1066666 - in /incubator/droids/branch/bchapuis:
droids-core/src/main/java/org/apache/droids/
droids-core/src/main/java/org/apache/droids/api/
droids-core/src/main/java/org/apache/droids/impl/
droids-core/src/main/java/org/apache/droids/rob...
Author: bchapuis
Date: Wed Feb 2 22:29:28 2011
New Revision: 1066666
URL: http://svn.apache.org/viewvc?rev=1066666&view=rev
Log:
applied droids-56-v2.patch on the branch
Modified:
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/AbstractDroid.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/Droid.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueue.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java
incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
incubator/droids/branch/bchapuis/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml
incubator/droids/branch/bchapuis/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
incubator/droids/branch/bchapuis/droids-spring/src/test/resources/droids-core-test-context.xml
incubator/droids/branch/bchapuis/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java
incubator/droids/branch/bchapuis/droids-wicket/src/main/resources/applicationContext.xml
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/AbstractDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/AbstractDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/AbstractDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/AbstractDroid.java Wed Feb 2 22:29:28 2011
@@ -16,10 +16,10 @@
*/
package org.apache.droids;
+import java.util.Queue;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Task;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.helper.Loggable;
/**
@@ -27,10 +27,10 @@ import org.apache.droids.helper.Loggable
*/
public abstract class AbstractDroid<T extends Task> extends Loggable implements Droid<T>
{
- protected final TaskQueue<T> queue;
+ protected final Queue<T> queue;
protected final TaskMaster<T> taskMaster;
- public AbstractDroid( TaskQueue<T> queue, TaskMaster<T> taskMaster )
+ public AbstractDroid( Queue<T> queue, TaskMaster<T> taskMaster )
{
this.queue = queue;
this.taskMaster = taskMaster;
@@ -38,10 +38,10 @@ public abstract class AbstractDroid<T ex
public void start()
{
- taskMaster.processAllTasks(queue, this);
+ taskMaster.start(queue, this);
}
- public TaskQueue<T> getQueue() {
+ public Queue<T> getQueue() {
return queue;
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/Droid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/Droid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/Droid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/Droid.java Wed Feb 2 22:29:28 2011
@@ -16,6 +16,7 @@
*/
package org.apache.droids.api;
+import java.util.Queue;
import org.apache.droids.exception.DroidsException;
/**
@@ -28,7 +29,8 @@ import org.apache.droids.exception.Droid
*
* @version 1.0
*/
-public interface Droid<T extends Task> {
+public interface Droid<T extends Task>
+{
/**
* Initialize the queue. Can have different implementation but the main groups
@@ -51,12 +53,19 @@ public interface Droid<T extends Task> {
* Invoke when the droid has completed
*/
void finished();
-
+
+ /**
+ * Return the tasks queue
+ *
+ * @return
+ */
+ public Queue<T> getQueue();
+
/**
* Ask the droid for a new worker
*/
Worker<T> getNewWorker();
-
+
/**
* Get the task master
*/
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskMaster.java Wed Feb 2 22:29:28 2011
@@ -17,31 +17,58 @@
package org.apache.droids.api;
import java.util.Date;
+import java.util.Queue;
import java.util.concurrent.TimeUnit;
-
-
/**
* Responsible for running all the tasks
*/
-public interface TaskMaster<T extends Task> {
- public enum ExecutionState {
- INITALIZED,
- RUNNING,
- COMPLETE
+public interface TaskMaster<T extends Task>
+{
+
+ public enum ExecutionState
+ {
+ RUNNING, STOPPED, COMPLETED
};
-
- void processAllTasks( final TaskQueue<T> queue, final Droid<T> droid );
- Date getStartTime();
+ /**
+ * Start the process of running tasks.
+ * l
+ * @param droid
+ */
+ void start(final Queue<T> queue, final Droid<T> droid);
+
+ /**
+ * Stops the process of running tasks.
+ */
+ void stop();
+
+ /**
+ * Blocks until all tasks have completed execution.
+ *
+ * @param timeout
+ * @param unit
+ * @return
+ * @throws InterruptedException
+ */
+ boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException;
- Date getFinishedWorking();
+ void setExceptionHandler(TaskExceptionHandler taskExceptionHandler);
+ void setDelayTimer(DelayTimer simpleDelayTimer);
+
+ /**
+ * Returns the current state.
+ *
+ * @return
+ */
ExecutionState getExecutionState();
+ Date getStartTime();
+
+ Date getFinishedWorking();
+
long getCompletedTasks();
T getLastCompletedTask();
-
- boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException;
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/api/TaskQueue.java Wed Feb 2 22:29:28 2011
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.api;
-
-import java.util.Collection;
-
-import org.apache.droids.exception.InvalidTaskException;
-
-
-/**
- * A queue is the data structure where the different tasks are waiting for
- * service.
- * <p>
- * Droids use this object to determine the next task to process and the
- * overall workload.
- *
- * @version 1.0
- *
- */
-public interface TaskQueue<T extends Task> {
- /**
- * Return the task that is identified with the given id
- *
- * @param id
- * of the task we want
- * @return the task idenfied by the given id
- */
- T getTask(String id);
-
- /**
- * Do we have more task waiting for service
- *
- * @return true if we have. false otherwise
- */
- boolean hasNext();
-
- /**
- * Return the next task that is waiting for service
- *
- * @return next task in line.
- */
- T next();
-
- /**
- * Add a Task to the queue...
- *
- * @param task task to add to the Queue
- * @throws InvalidTaskException
- */
- void merge(T task) throws InvalidTaskException;
-
- /**
- * Merge a given list of tasks with the current queue. Whether or not we
- * eliminate double entries is up to the implementation
- *
- * @param tasks
- * the list of task that we want to add to the queue
- * @throws InvalidTaskException
- */
- void merge(Collection<? extends T> tasks) throws InvalidTaskException;
-
- /**
- * How many task do we have <em>left</em> in the queue.
- *
- * @return number of tasks do we have <em>left</em> in the queue.
- */
- int getSize();
-
- /**
- * Get the top 'size' elements in the queue
- */
- Collection<? extends T> peek( int size );
-}
\ No newline at end of file
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/MultiThreadedTaskMaster.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/MultiThreadedTaskMaster.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/MultiThreadedTaskMaster.java Wed Feb 2 22:29:28 2011
@@ -1,401 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.impl;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.droids.api.DelayTimer;
-import org.apache.droids.api.Droid;
-import org.apache.droids.api.Task;
-import org.apache.droids.api.TaskExceptionHandler;
-import org.apache.droids.api.TaskExceptionResult;
-import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
-import org.apache.droids.api.WorkMonitor;
-import org.apache.droids.api.Worker;
-import org.apache.droids.exception.DroidsException;
-
-/**
- * This task master provides a base implementation that support multithreaded task processing powered by a
- * ThreadPoolExecutor.
- * <p/>
- * By default, the ThreadPoolExecutor uses a bounded blocking queue with a size as same as the maxThreads. As long as
- * there is any outstanding task in the Task Queue and the Executor has spare capacity, a new Thread will be created to
- * poll and handle one task from the Task Queue. User may set to use another 'pool' implementation, and the nextTask()
- * method may be overriden by sub-class.
- * <p/>
- * This Task Master doesn't support: pausing, monitoring etc. TaskExceptionHandler is unimplemented.
- * <p/>
- * And there is no failure handling mechanism, e.g. if the JVM of Task Master is crashed, any task polled from the Task Queue
- * will be loss. (this point makes sense only if the Task Queue is persistent)
- *
- * @param <T>
- */
-public class MultiThreadedTaskMaster<T extends Task> implements TaskMaster<T> {
- protected final Log log = LogFactory.getLog(MultiThreadedTaskMaster.class);
- private static final long KEEP_ALIVE = 50000L;
- private static final long TICKLE_TIME = 250L;
-
- protected String name = getClass().getName();
- protected ThreadPoolExecutor pool = null;
- protected int maxThreads = 0;
- protected TaskQueue<T> queue = null;
- protected Droid<T> droid = null;
- protected DelayTimer delayTimer = null;
- protected WorkMonitor<T> monitor = null;
-
- protected Date startedWorking = null;
- protected Date finishedWorking = null;
- protected T lastCompletedTask = null;
- protected volatile ExecutionState state = ExecutionState.INITALIZED;
- protected AtomicLong completedCount = new AtomicLong();
- protected TaskExceptionHandler exHandler;
-
-
- class DroidsThreadFactory implements ThreadFactory {
- final ThreadGroup group;
- final AtomicInteger threadNumber = new AtomicInteger(1);
- final String prefix;
-
- DroidsThreadFactory( String descr ) {
- prefix = (descr==null) ? name+"-" : name+"-"+descr+"-";
- SecurityManager s = System.getSecurityManager();
- group = (s != null)? s.getThreadGroup() :
- Thread.currentThread().getThreadGroup();
- }
-
- DroidsThreadFactory() {
- this( null );
- }
-
- public Thread newThread(Runnable r) {
- Thread t = new Thread(group, r,
- prefix + threadNumber.getAndIncrement(),
- 0);
- if (t.isDaemon())
- t.setDaemon(false);
- if (t.getPriority() != Thread.NORM_PRIORITY)
- t.setPriority(Thread.NORM_PRIORITY);
- return t;
- }
- }
-
- /**
- * The queue has been initialized
- */
- public void processAllTasks(final TaskQueue<T> queue, final Droid<T> droid) {
- if( state == ExecutionState.RUNNING ) {
- // TODO? throw an error?
- log.info("already processing tasks" );
- return;
- }
-
- this.queue = queue;
- this.droid = droid;
- this.startedWorking = new Date();
- this.finishedWorking = null;
- this.state = ExecutionState.RUNNING;
-
- if (!queue.hasNext()) {
- log.info("no tasks. finishing now");
- terminate();
- return;
- }
-
- // start the pool
- int poolSize = getMaxThreads();
- // user may set a thread pool before calling the processAllTasks
- this.pool = this.pool != null ? this.pool : new ThreadPoolExecutor(poolSize, poolSize, KEEP_ALIVE, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(poolSize), new DroidsThreadFactory( "runner" ) );
- this.pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());//task is not yet run, there is no impact to discard fail to submit task
- if (log.isInfoEnabled()) {
- log.info("processAllTasks() - created ThreadPoolExecutor - poolSize: " + poolSize);
- }
-
- // process tasks in a new thread
- ExecutorService executor = Executors.newSingleThreadExecutor( new DroidsThreadFactory( "main" ) );
- executor.execute( new Runnable() {
- @Override
- public void run() {
- while( state == ExecutionState.RUNNING && !pool.isTerminating() ) {
- if( queue.hasNext() ) {
- if( pool.getActiveCount() < pool.getMaximumPoolSize() ) {
- submitNewTaskRunner(pool, queue);
- continue; // skip the rest of the loop
- }
- else {
- // pool is full but we have more to do
- //log.info("More tasks to process, but pool is full");
- }
- }
- else {
- // no new tasks to process
- //log.info("No tasks to process....");
- }
-
- // Try sleeping for a little bit
- try {
- if (log.isDebugEnabled())
- log.debug("processAllTasks() - no task or thread pool is full, to check again in " + TICKLE_TIME + "ms - queue.size: " + queue.getSize());
- Thread.sleep(TICKLE_TIME);
- }
- catch (InterruptedException e) {
- log.error(e);
- }
-
- // Check if we are done
- if( !queue.hasNext() && pool.getActiveCount() < 1 ) {
- log.info("Finshed invocation, waiting for workers to finish.");
- terminate();
- break;
- }
- }
- }
- });
- }
-
- /**
- * This method is designed for sub-classing
- * <p/>
- * TODO: refactor and create a specialized thread that
- * - construct with the delayTimer and transparently apply the delay
- * - construct with the Task and transparent set the lastCompleteTask
- * <p/>
- * TODO: consider to provide all variables as argument so that the method overrider needs not to call the parent
- * variables
- * <p/>
- * TODO: caller of this method could use the Future to track the task completion and perform the termination and
- * other checking.
- *
- * @param executor
- * @param queue
- * @return
- */
- protected Future<?> submitNewTaskRunner(ExecutorService executor, final TaskQueue<? extends T> queue) {
- return executor.submit(new Runnable() {
- public void run() {
- String threadName = Thread.currentThread().getName();
-
- // use the same thread to keep processing
- while( queue.hasNext() && pool.getActiveCount() <= maxThreads && !pool.isTerminating() ) {
- T task = queue.next();
-
- long delay = (delayTimer != null) ? delayTimer.getDelayMillis() : 0;
- if (log.isDebugEnabled())
- log.debug("run() - begin - thread: " + threadName + ", task: " + task.getId() + ", delay: " + delay);
-
- Worker<T> worker = null;
- Exception ex = null;
- try {
- if( delay > 0 ) {
- Thread.sleep(delay); // gets the current thread
- }
- worker = droid.getNewWorker();
- if( monitor != null ) {
- monitor.beforeExecute( task, worker );
- }
- if(!task.isAborted())
- worker.execute(task);
- lastCompletedTask = task;
- } catch (DroidsException e) {
- ex = e;
- } catch (IOException e) {
- ex = e;
- } catch (InterruptedException e) {
- ex = e;
- }
- finally {
- // Handle any exceptions
- boolean terminate = false;
- if (ex != null) {
- try {
- TaskExceptionResult result = TaskExceptionResult.WARN;
- if (exHandler != null) {
- result = exHandler.handleException(ex);
- }
- switch (result) {
- case WARN:
- log.warn(ex.toString(), ex);
- break;
- case FATAL:
- log.warn(ex.getMessage(), ex);
- terminate = true;
- break;
- case IGNORE: break; // nothing
- }
- }
- catch( Exception e2 ) {
- log.error( e2.getMessage(), e2 );
- }
- }
-
- if( monitor != null ) {
- monitor.afterExecute( task, worker, ex );
- }
-
- completedCount.incrementAndGet();
- if (log.isInfoEnabled()) {
- log.info("run() - done - completedCount: " + completedCount + ", thread: " + threadName +
- ", task: " + task.getId() + ", queue.size: " + queue.getSize() +
- ", pool.activeCount: " + pool.getActiveCount());
- }
-
- if( terminate ) {
- terminate();
- }
- } // loop
- }
- }
- });
- }
-
-
- private void terminate() {
- if (pool != null) {
- shutdownAndAwaitTermination();
- }
-
- long elapsed = System.currentTimeMillis() - startedWorking.getTime();
- if (log.isInfoEnabled()) {
- log.info("All threads have finished. (elapsed: " + elapsed + ")");
- }
- finishedWorking = new Date();
- state = ExecutionState.COMPLETE;
- droid.finished();
- }
-
- /**
- * Set the maximum allowed thread count. If the new value is less
- * than the current value, excess existing threads will be terminated
- * when they become idle.
- *
- * @param count
- */
- public void setMaxThreads( int count ) {
- if( pool != null ) {
- pool.setMaximumPoolSize( count );
-
- // NOTE, the main loop should fill up any spaces
- // within TICKLE_TIME
- }
- maxThreads = count;
- }
-
- /**
- * Get number of maximum allowed threads
- *
- * @return the number of maximum threads that we allow
- */
- public int getMaxThreads() {
- return maxThreads;
- }
-
- public ExecutionState getExecutionState() {
- return state;
- }
-
- /**
- * Shutdown all threads, close the pools and leave.
- * If it is not working by asking nice to shutdown just kill all
- * threads.
- */
- public void shutdownAndAwaitTermination() {
- log.info("SHUTTING DOWN");
- if (pool == null) {
- return;
- }
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
- log.info("SHUT DOWN NOW");
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for to respond to being canceled
- if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
- log.info("Pool did not terminate");
- }
- }
- }
- catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
-
- public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
- log.info("awaitTermination()");
- return pool.awaitTermination(timeout, unit);
- }
-
- public final void setExceptionHandler(TaskExceptionHandler exHandler) {
- this.exHandler = exHandler;
- }
-
- public DelayTimer getDelayTimer() {
- return delayTimer;
- }
-
- public void setDelayTimer(DelayTimer delayTimer) {
- this.delayTimer = delayTimer;
- }
-
- public long getCompletedTasks() {
- return completedCount.get();
- }
-
- public Date getFinishedWorking() {
- return finishedWorking;
- }
-
- public T getLastCompletedTask() {
- return lastCompletedTask;
- }
-
- public Date getStartTime() {
- return startedWorking;
- }
-
- public WorkMonitor<T> getMonitor() {
- return monitor;
- }
-
- public void setMonitor(WorkMonitor<T> monitor) {
- this.monitor = monitor;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java Wed Feb 2 22:29:28 2011
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.impl;
-
-import java.util.Date;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.droids.api.DelayTimer;
-import org.apache.droids.api.Droid;
-import org.apache.droids.api.Task;
-import org.apache.droids.api.TaskExceptionHandler;
-import org.apache.droids.api.TaskExceptionResult;
-import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
-import org.apache.droids.api.Worker;
-import org.apache.droids.helper.Loggable;
-
-public class SequentialTaskMaster<T extends Task>
- extends Loggable implements TaskMaster<T>
-{
- private final Object mutex;
-
- private volatile boolean completed;
- private volatile Date startedWorking = null;
- private volatile Date finishedWorking = null;
- private volatile int completedTask = 0;
- private volatile T lastCompletedTask = null;
- private volatile ExecutionState state = ExecutionState.INITALIZED;
-
- private DelayTimer delayTimer = null;
- private TaskExceptionHandler exHandler = null;
-
- public SequentialTaskMaster() {
- super();
- this.mutex = new Object();
- }
-
- /**
- * The queue has been initialized
- */
- public synchronized void processAllTasks(
- final TaskQueue<T> queue, final Droid<T> droid)
- {
- this.completed = false;
- this.startedWorking = new Date();
- this.finishedWorking = null;
- this.completedTask = 0;
- this.state = ExecutionState.RUNNING;
-
- boolean terminated = false;
- while( !terminated ) {
- T task = queue.next();
- if (task == null) {
- break;
- }
- if( delayTimer != null ) {
- long delay = delayTimer.getDelayMillis();
- if( delay > 0 ) {
- try {
- Thread.sleep( delay );
- }
- catch (InterruptedException e) {}
- }
- }
- Worker<T> worker = droid.getNewWorker();
- try {
- if(!task.isAborted())
- worker.execute( task );
- completedTask++;
- lastCompletedTask = task;
- } catch (Exception ex) {
- TaskExceptionResult result = TaskExceptionResult.WARN;
- if (exHandler != null) {
- result = exHandler.handleException(ex);
- }
- switch (result) {
- case WARN:
- log.warn(ex.toString());
- break;
- case FATAL:
- log.warn(ex.getMessage(), ex);
- terminated = true;
- break;
- }
- }
- }
- finishedWorking = new Date();
- this.state = ExecutionState.COMPLETE;
- droid.finished();
- synchronized( mutex ) {
- completed = true;
- mutex.notifyAll();
- }
- }
-
- public final void setExceptionHandler(TaskExceptionHandler exHandler) {
- this.exHandler = exHandler;
- }
-
-
- public final void setDelayTimer(DelayTimer delayTimer) {
- this.delayTimer = delayTimer;
- }
-
- public boolean isWorking()
- {
- return startedWorking != null && finishedWorking == null;
- }
-
- public Date getStartTime() {
- return startedWorking;
- }
-
-
- public Date getFinishedWorking() {
- return finishedWorking;
- }
-
-
- public long getCompletedTasks() {
- return completedTask;
- }
-
-
- public T getLastCompletedTask() {
- return lastCompletedTask;
- }
-
-
- public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
- if (timeout < 0) {
- timeout = 0;
- }
- synchronized (this.mutex) {
- long deadline = System.currentTimeMillis() + unit.toMillis(timeout);
- long remaining = timeout;
- while (!completed) {
- this.mutex.wait(remaining);
- if (timeout >= 0) {
- remaining = deadline - System.currentTimeMillis();
- if (remaining <= 0) {
- return false; // Reach if timeout is over and no finish.
- }
- }
- }
- }
- return true;
- }
-
- @Override
- public ExecutionState getExecutionState() {
- return state;
- }
-}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueue.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueue.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueue.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueue.java Wed Feb 2 22:29:28 2011
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.impl;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import org.apache.droids.api.*;
-import org.apache.droids.exception.InvalidTaskException;
-
-/**
- * A simple
- */
-public class SimpleTaskQueue<T extends Task> implements TaskQueue<T>, TaskValidator<T> {
- private final Queue<T> queue;
- private final Map<String,T> taskMap;
-
- private TaskValidator<T> taskValidator;
- private int maxSize = 0;
-
- /**
- * Simple queue constructor.
- */
- public SimpleTaskQueue() {
- queue = new ConcurrentLinkedQueue<T>();
- taskMap = new ConcurrentHashMap<String, T>();
- }
-
- public SimpleTaskQueue(Queue<T> queue) {
- this.queue = queue;
- taskMap = new ConcurrentHashMap<String, T>();
- }
-
- /*
- * @see org.apache.droids.api.Queue#getTask(java.lang.String)
- */
- public T getTask(String id) {
- return taskMap.get(id);
- }
-
- /*
- * @see org.apache.droids.api.Queue#totalSize()
- */
- public int getSize() {
- return queue.size();
- }
-
-
- public boolean hasNext() {
- return !queue.isEmpty();
- }
-
- public void merge(T task) throws InvalidTaskException {
- // if we alreay have the id in our map, then don't add it to the queue...
- synchronized( taskMap ) {
- task = this.validateTask( task );
- String key = task.getId();
- if( !taskMap.containsKey( key ) ) {
- queue.add( task );
- taskMap.put(key, task);
- }
- }
- }
-
- /**
- * Tasks with duplicate keys will be overwritten
- * @throws InvalidTaskException
- */
- public void merge(Collection<? extends T> tasks) throws InvalidTaskException {
- for( T task : tasks ) {
- merge( task );
- }
- }
-
- public T next() {
- T task = queue.poll();
- if( task != null ) {
- taskMap.remove( task.getId() );
- }
- return task;
- }
-
- public void clear()
- {
- synchronized( this ) {
- queue.clear();
- taskMap.clear();
- }
- }
-
- public T validateTask(T task) throws InvalidTaskException {
- if( taskValidator != null ) {
- task = taskValidator.validateTask( task );
- }
- /*if( maxSize > 0 && queue.size() >= maxSize ) {
- throw new InvalidTaskException( "queue is full: "+queue.size() );
- }*/
- return task;
- }
-
- //------------------------------------------------------
- //------------------------------------------------------
-
- public TaskValidator<T> getTaskValidator() {
- return taskValidator;
- }
-
- public void setTaskValidator(TaskValidator<T> taskValidator) {
- this.taskValidator = taskValidator;
- }
-
- public int getMaxSize() {
- return maxSize;
- }
-
- public void setMaxSize(int maxSize) {
- this.maxSize = maxSize;
- }
-
- @Override
- public Collection<? extends T> peek(int size) {
- ArrayList<T> vals = new ArrayList<T>( size );
- Iterator<T> iter = queue.iterator();
- while( iter.hasNext() && vals.size() < size ) {
- vals.add( iter.next() );
- }
- return vals;
- }
-}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/impl/SimpleTaskQueueWithHistory.java Wed Feb 2 22:29:28 2011
@@ -16,36 +16,65 @@
*/
package org.apache.droids.impl;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedList;
import org.apache.droids.api.Task;
-import org.apache.droids.exception.InvalidTaskException;
/**
* Extend the task queue to ignore any tasks we have already seen
*/
-public class SimpleTaskQueueWithHistory<T extends Task> extends SimpleTaskQueue<T> {
+public class SimpleTaskQueueWithHistory<T extends Task> extends LinkedList<T>
+{
+
private final java.util.Set<String> previous;
/**
* Simple queue constructor.
*/
- public SimpleTaskQueueWithHistory() {
+ public SimpleTaskQueueWithHistory()
+ {
super();
- previous = Collections.synchronizedSet( new HashSet<String>() );
+ previous = Collections.synchronizedSet(new HashSet<String>());
+ }
+
+ @Override
+ public boolean offer(T e)
+ {
+ if (previous.add(e.getId())) {
+ return super.offer(e);
+ } else {
+ return false;
+ }
}
@Override
- public void merge(T task) throws InvalidTaskException {
- // only add it if we have not seen the taskId before...
- if( previous.add( task.getId() ) ) {
- super.merge( task );
+ public boolean add(T e)
+ {
+ if (previous.add(e.getId())) {
+ return super.offer(e);
+ } else {
+ return false;
}
}
-
+
@Override
- public void clear()
+ public boolean addAll(Collection<? extends T> c)
+ {
+ HashSet<String> ids = new HashSet<String>();
+ for (T e : c) {
+ ids.add(e.getId());
+ }
+ if (previous.addAll(ids)) {
+ return super.addAll(c);
+ } else {
+ return false;
+ }
+ }
+
+ public void clearHistory()
{
previous.clear();
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java Wed Feb 2 22:29:28 2011
@@ -19,12 +19,12 @@ package org.apache.droids.robot.crawler;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collection;
+import java.util.Queue;
import org.apache.droids.AbstractDroid;
import org.apache.droids.LinkTask;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.exception.InvalidTaskException;
import org.apache.droids.helper.factories.ParserFactory;
@@ -39,7 +39,7 @@ public abstract class CrawlingDroid exte
ParserFactory parserFactory;
URLFiltersFactory filtersFactory;
- public CrawlingDroid( TaskQueue<Link> queue, TaskMaster<Link> taskMaster )
+ public CrawlingDroid( Queue<Link> queue, TaskMaster<Link> taskMaster )
{
super( queue, taskMaster );
}
@@ -59,13 +59,13 @@ public abstract class CrawlingDroid exte
} catch (URISyntaxException ex) {
throw new InvalidTaskException("Invalid lication: " + location);
}
- queue.merge( new LinkTask( null, uri, 0 ) );
+ queue.offer( new LinkTask( null, uri, 0 ) );
}
}
public void start()
{
- taskMaster.processAllTasks(queue, this);
+ taskMaster.start(queue, this);
}
public void finished()
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java Wed Feb 2 22:29:28 2011
@@ -83,7 +83,7 @@ public class CrawlingWorker extends Logg
Parse parse = parser.parse(entity, link);
if( parse.getOutlinks() != null ) {
Collection<Link> outlinks = getFilteredOutlinks( parse );
- droid.getQueue().merge( outlinks );
+ droid.getQueue().addAll( outlinks );
}
entity.setParse(parse);
handle(entity, link);
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/FileWorker.java Wed Feb 2 22:29:28 2011
@@ -18,16 +18,16 @@ package org.apache.droids.robot.walker;
import java.io.File;
-import org.apache.droids.api.TaskQueue;
+import java.util.Queue;
import org.apache.droids.api.Worker;
import org.apache.droids.exception.InvalidTaskException;
import org.apache.droids.helper.Loggable;
public class FileWorker extends Loggable implements Worker<FileTask>
{
- final TaskQueue<FileTask> queue;
+ final Queue<FileTask> queue;
- public FileWorker( TaskQueue<FileTask> queue )
+ public FileWorker( Queue<FileTask> queue )
{
this.queue = queue;
}
@@ -39,7 +39,7 @@ public class FileWorker extends Loggable
File[] files = file.listFiles();
if( files != null ) {
for( File f : files ) {
- queue.merge( new FileTask( f, task.getDepth()+1 ) );
+ queue.add( new FileTask( f, task.getDepth() + 1 ) );
}
}
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/main/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java Wed Feb 2 22:29:28 2011
@@ -19,19 +19,20 @@ package org.apache.droids.robot.walker;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.LinkedList;
+import java.util.Queue;
import java.util.concurrent.TimeUnit;
import org.apache.droids.AbstractDroid;
import org.apache.droids.exception.InvalidTaskException;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
import org.apache.droids.api.*;
+import org.apache.droids.impl.TaskMasterImpl;
public class SimpleWalkingDroid extends AbstractDroid<FileTask> implements WalkingDroid
{
private Collection<File> initialFiles;
- public SimpleWalkingDroid( TaskQueue<FileTask> queue, TaskMaster<FileTask> taskMaster )
+ public SimpleWalkingDroid( Queue<FileTask> queue, TaskMaster<FileTask> taskMaster )
{
super( queue, taskMaster );
}
@@ -45,7 +46,7 @@ public class SimpleWalkingDroid extends
throw new IllegalStateException( "FileSystemWalker requires at least one starting file" );
}
for( File file : initialFiles ) {
- queue.merge( new FileTask( file, 0 ) );
+ queue.add( new FileTask( file, 0 ) );
}
}
@@ -63,10 +64,10 @@ public class SimpleWalkingDroid extends
public static void main( String[] args ) throws Exception
{
- MultiThreadedTaskMaster<FileTask> taskMaster = new MultiThreadedTaskMaster<FileTask>();
- taskMaster.setMaxThreads( 3 );
+ TaskMasterImpl<FileTask> taskMaster = new TaskMasterImpl<FileTask>();
+ taskMaster.setPoolSize(3);
- TaskQueue<FileTask> queue = new SimpleTaskQueue<FileTask>();
+ Queue<FileTask> queue = new LinkedList<FileTask>();
Collection<File> files = new ArrayList<File>();
files.add( new File( args[0] ) ); //APACHE/droids/src" ) );
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/DroidsFactory.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/DroidsFactory.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/DroidsFactory.java Wed Feb 2 22:29:28 2011
@@ -16,156 +16,161 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.droids;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-
-import org.apache.droids.api.Droid;
-import org.apache.droids.api.Handler;
-import org.apache.droids.api.Link;
-import org.apache.droids.api.URLFilter;
-import org.apache.droids.delay.SimpleDelayTimer;
+package org.apache.droids;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Queue;
+
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Handler;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.URLFilter;
+import org.apache.droids.delay.SimpleDelayTimer;
import org.apache.droids.handle.SaveHandler;
-import org.apache.droids.helper.factories.HandlerFactory;
-import org.apache.droids.helper.factories.ParserFactory;
-import org.apache.droids.helper.factories.ProtocolFactory;
-import org.apache.droids.helper.factories.URLFiltersFactory;
-import org.apache.droids.impl.DefaultTaskExceptionHandler;
-import org.apache.droids.impl.ExceptionCrawlingDroid;
-import org.apache.droids.impl.ReportCrawlingDroid;
-import org.apache.droids.impl.SaveCrawlingDroid;
-import org.apache.droids.impl.SequentialTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
-import org.apache.droids.impl.SimpleTaskQueueWithHistory;
-import org.apache.droids.parse.html.HtmlParser;
-import org.apache.droids.protocol.http.HttpProtocol;
-import org.apache.droids.robot.crawler.CrawlingDroid;
-
-public class DroidsFactory
-{
-
- public static ParserFactory createDefaultParserFactory() {
- ParserFactory parserFactory = new ParserFactory();
- HtmlParser htmlParser = new HtmlParser();
- htmlParser.setElements(new HashMap<String, String>());
- htmlParser.getElements().put("a", "href");
- htmlParser.getElements().put("link", "href");
- htmlParser.getElements().put("img", "src");
- htmlParser.getElements().put("script", "src");
- parserFactory.setMap(new HashMap<String, Object>());
- parserFactory.getMap().put("text/html", htmlParser);
- return parserFactory;
- }
-
- public static ProtocolFactory createDefaultProtocolFactory() {
- ProtocolFactory protocolFactory = new ProtocolFactory();
- HttpProtocol httpProtocol = new HttpProtocol();
- httpProtocol.setForceAllow(true);
-
- protocolFactory.setMap(new HashMap<String, Object>());
- protocolFactory.getMap().put("http", httpProtocol);
- return protocolFactory;
- }
-
- public static URLFiltersFactory createDefaultURLFiltersFactory() {
- URLFiltersFactory filtersFactory = new URLFiltersFactory();
- URLFilter defaultURLFilter = new URLFilter() {
-
- public String filter(String urlString) {
- return urlString;
- }
-
- };
- filtersFactory.setMap(new HashMap<String, Object>());
- filtersFactory.getMap().put("default", defaultURLFilter);
- return filtersFactory;
- }
-
- public static HandlerFactory createDefaultHandlerFactory(Handler defaultHandler) {
- HandlerFactory handlerFactory = new HandlerFactory();
- handlerFactory.setMap(new HashMap<String, Object>());
- handlerFactory.getMap().put("default", defaultHandler);
- return handlerFactory;
- }
-
- public static Droid<Link> createSimpleSaveCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SimpleTaskQueueWithHistory<Link> simpleQueue = new SimpleTaskQueueWithHistory<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
+import org.apache.droids.helper.factories.HandlerFactory;
+import org.apache.droids.helper.factories.ParserFactory;
+import org.apache.droids.helper.factories.ProtocolFactory;
+import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.impl.DefaultTaskExceptionHandler;
+import org.apache.droids.impl.TaskMasterImpl;
+import org.apache.droids.impl.ExceptionCrawlingDroid;
+import org.apache.droids.impl.ReportCrawlingDroid;
+import org.apache.droids.impl.SaveCrawlingDroid;
+import org.apache.droids.impl.SimpleTaskQueueWithHistory;
+import org.apache.droids.parse.html.HtmlParser;
+import org.apache.droids.protocol.http.HttpProtocol;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+
+public class DroidsFactory
+{
+
+ public static ParserFactory createDefaultParserFactory() {
+ ParserFactory parserFactory = new ParserFactory();
+ HtmlParser htmlParser = new HtmlParser();
+ htmlParser.setElements(new HashMap<String, String>());
+ htmlParser.getElements().put("a", "href");
+ htmlParser.getElements().put("link", "href");
+ htmlParser.getElements().put("img", "src");
+ htmlParser.getElements().put("script", "src");
+ parserFactory.setMap(new HashMap<String, Object>());
+ parserFactory.getMap().put("text/html", htmlParser);
+ return parserFactory;
+ }
+
+ public static ProtocolFactory createDefaultProtocolFactory() {
+ ProtocolFactory protocolFactory = new ProtocolFactory();
+ HttpProtocol httpProtocol = new HttpProtocol();
+ httpProtocol.setForceAllow(true);
+
+ protocolFactory.setMap(new HashMap<String, Object>());
+ protocolFactory.getMap().put("http", httpProtocol);
+ return protocolFactory;
+ }
+
+ public static URLFiltersFactory createDefaultURLFiltersFactory() {
+ URLFiltersFactory filtersFactory = new URLFiltersFactory();
+ URLFilter defaultURLFilter = new URLFilter() {
+
+ public String filter(String urlString) {
+ return urlString;
+ }
+
+ };
+ filtersFactory.setMap(new HashMap<String, Object>());
+ filtersFactory.getMap().put("default", defaultURLFilter);
+ return filtersFactory;
+ }
+
+ public static HandlerFactory createDefaultHandlerFactory(Handler defaultHandler) {
+ HandlerFactory handlerFactory = new HandlerFactory();
+ handlerFactory.setMap(new HashMap<String, Object>());
+ handlerFactory.getMap().put("default", defaultHandler);
+ return handlerFactory;
+ }
+
+ public static Droid<Link> createSimpleSaveCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ SimpleTaskQueueWithHistory<Link> simpleQueue = new SimpleTaskQueueWithHistory<Link>();
+
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster, new SaveHandler() );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
- public static Droid<Link> createSimpleReportCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- SimpleTaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-
- CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
- public static Droid<Link> createSimpleExceptionCrawlingDroid(
- String targetURI) {
- ParserFactory parserFactory = createDefaultParserFactory();
- ProtocolFactory protocolFactory = createDefaultProtocolFactory();
- URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
-
- SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
- simpleDelayTimer.setDelayMillis(100);
-
- SimpleTaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
- taskMaster.setDelayTimer( simpleDelayTimer );
- taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
-
- CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
- crawler.setFiltersFactory(filtersFactory);
- crawler.setParserFactory(parserFactory);
- crawler.setProtocolFactory(protocolFactory);
-
- Collection<String> initialLocations = new ArrayList<String>();
- initialLocations.add( targetURI );
- crawler.setInitialLocations(initialLocations);
- return crawler;
- }
-
-}
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleReportCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+// SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
+// MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
+ Queue<Link> queue = new LinkedList<Link>();
+
+ CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleExceptionCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ Queue<Link> queue = new LinkedList<Link>();
+
+// SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
+// MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
+ CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+}
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/FileRenameDroid.java Wed Feb 2 22:29:28 2011
@@ -20,13 +20,13 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.Queue;
import org.apache.droids.AbstractDroid;
import org.apache.droids.api.*;
-import org.apache.droids.exception.InvalidTaskException;
+import org.apache.droids.impl.TaskMasterImpl;
import org.apache.droids.robot.walker.FileTask;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
import org.apache.log4j.Logger;
@@ -36,7 +36,7 @@ public class FileRenameDroid extends Abs
private Collection<File> initialFiles;
- public FileRenameDroid( TaskQueue<FileTask> queue, TaskMaster<FileTask> taskMaster )
+ public FileRenameDroid( Queue<FileTask> queue, TaskMaster<FileTask> taskMaster )
{
super( queue, taskMaster );
}
@@ -49,14 +49,11 @@ public class FileRenameDroid extends Abs
if( initialFiles == null || initialFiles.isEmpty() ) {
throw new RuntimeException( "FileSystemWalker requires at least one starting file" );
}
- try {
+
for( File file : initialFiles ) {
- queue.merge( new FileTask( file, 0 ) );
+ queue.add( new FileTask( file, 0 ) );
}
- }
- catch (InvalidTaskException e) {
- e.printStackTrace();
- }
+
}
@@ -128,10 +125,10 @@ public class FileRenameDroid extends Abs
public static void main( String[] args )
{
- MultiThreadedTaskMaster<FileTask> taskMaster = new MultiThreadedTaskMaster<FileTask>();
- taskMaster.setMaxThreads( 3 );
+ TaskMasterImpl<FileTask> taskMaster = new TaskMasterImpl<FileTask>();
+ taskMaster.setPoolSize( 3 );
- TaskQueue<FileTask> queue = new SimpleTaskQueue<FileTask>();
+ Queue<FileTask> queue = new LinkedList<FileTask>();
Collection<File> files = new ArrayList<File>();
files.add( new File( args[0] ) );
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java Wed Feb 2 22:29:28 2011
@@ -19,6 +19,8 @@ package org.apache.droids.examples;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Queue;
import java.util.concurrent.TimeUnit;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -31,8 +33,7 @@ import org.apache.droids.helper.factorie
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
import org.apache.droids.impl.DefaultTaskExceptionHandler;
-import org.apache.droids.impl.SequentialTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.droids.impl.TaskMasterImpl;
import org.apache.droids.impl.SysoutCrawlingDroid;
import org.apache.droids.net.RegexURLFilter;
import org.apache.droids.parse.html.HtmlParser;
@@ -124,11 +125,9 @@ public class SimpleRuntime {
SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
simpleDelayTimer.setDelayMillis(100);
- SimpleTaskQueue<Link> simpleQueue = new SimpleTaskQueue<Link>();
- // simpleQueue.setMaxDepth(3);
- // simpleQueue.setMaxSize(-1);
+ Queue<Link> simpleQueue = new LinkedList<Link>();
- SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ TaskMasterImpl<Link> taskMaster = new TaskMasterImpl<Link>();
taskMaster.setDelayTimer( simpleDelayTimer );
taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java Wed Feb 2 22:29:28 2011
@@ -18,11 +18,11 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.examples.handler.ExceptionReportHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -30,7 +30,7 @@ import org.apache.droids.robot.crawler.C
public class ExceptionCrawlingDroid extends CrawlingDroid {
- public ExceptionCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
+ public ExceptionCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
super(queue, taskMaster);
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java Wed Feb 2 22:29:28 2011
@@ -18,10 +18,10 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.examples.handler.ReportHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -29,7 +29,7 @@ import org.apache.droids.robot.crawler.C
public class ReportCrawlingDroid extends CrawlingDroid {
- public ReportCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
+ public ReportCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
super(queue, taskMaster);
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java Wed Feb 2 22:29:28 2011
@@ -18,11 +18,11 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.robot.crawler.CrawlingDroid;
import org.apache.droids.robot.crawler.CrawlingWorker;
@@ -30,7 +30,7 @@ import org.apache.droids.robot.crawler.C
public class SaveCrawlingDroid extends CrawlingDroid {
private final Handler defaultHandler;
- public SaveCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster, final Handler defaultHandlerForWorkerCreation ) {
+ public SaveCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster, final Handler defaultHandlerForWorkerCreation ) {
super(queue, taskMaster);
assert( defaultHandlerForWorkerCreation != null );
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java Wed Feb 2 22:29:28 2011
@@ -18,10 +18,10 @@
*/
package org.apache.droids.impl;
+import java.util.Queue;
import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
import org.apache.droids.api.Worker;
import org.apache.droids.handle.SysoutHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
@@ -29,7 +29,7 @@ import org.apache.droids.robot.crawler.C
public class SysoutCrawlingDroid extends CrawlingDroid {
- public SysoutCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
+ public SysoutCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
super(queue, taskMaster);
}
Modified: incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java Wed Feb 2 22:29:28 2011
@@ -58,11 +58,12 @@ public class TestSimpleDroid
String targetURI = baseURI + "/start_html";
- Droid<Link> droid = DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
+ Droid<Link> droid = DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
droid.init();
droid.start();
- droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
+
+ while (!droid.getTaskMaster().awaitTermination(250L, TimeUnit.MILLISECONDS));
Assert.assertFalse(ReportHandler.getReport().isEmpty());
Assert.assertEquals(5, ReportHandler.getReport().size());
@@ -87,7 +88,7 @@ public class TestSimpleDroid
Droid<Link> droid = DroidsFactory.createSimpleExceptionCrawlingDroid(
targetURI);
- SequentialTaskMaster<Link> taskMaster = (SequentialTaskMaster<Link>) droid.getTaskMaster();
+ TaskMasterImpl<Link> taskMaster = (TaskMasterImpl<Link>) droid.getTaskMaster();
taskMaster.setExceptionHandler(new TaskExceptionHandler() {
public TaskExceptionResult handleException(Exception ex) {
@@ -96,12 +97,12 @@ public class TestSimpleDroid
}
return TaskExceptionResult.WARN;
}
-
+
});
droid.init();
droid.start();
- droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
+ while (!droid.getTaskMaster().awaitTermination(250L, TimeUnit.MILLISECONDS));
Assert.assertFalse(ReportHandler.getReport().isEmpty());
Assert.assertEquals(4, ReportHandler.getReport().size());
Modified: incubator/droids/branch/bchapuis/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml (original)
+++ incubator/droids/branch/bchapuis/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml Wed Feb 2 22:29:28 2011
@@ -44,7 +44,7 @@
</bean>
<bean name="taskMaster"
- class="org.apache.droids.impl.SequentialTaskMaster">
+ class="org.apache.droids.impl.TaskMasterImpl">
<property name="exceptionHandler" ref="taskExceptionHandler" />
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
<!--<property name="maxThreads" value="${droids.maxThreads}"/>-->
@@ -53,17 +53,16 @@
<!-- Droids -->
<bean name="org.apache.droids.api.Droid/hello"
class="org.apache.droids.impl.SysoutCrawlingDroid">
+ <constructor-arg ref="java.util.LinkedList" />
<constructor-arg ref="taskMaster" />
- <constructor-arg ref="org.apache.droids.impl.SimpleTaskQueue" />
<property name="protocolFactory" ref="org.apache.droids.helper.factories.ProtocolFactory"/>
<property name="parserFactory" ref="org.apache.droids.helper.factories.ParserFactory"/>
<property name="filtersFactory" ref="org.apache.droids.helper.factories.URLFiltersFactory"/>
</bean>
<!-- Queue -->
- <bean id="org.apache.droids.impl.SimpleTaskQueue"
- class="org.apache.droids.impl.SimpleTaskQueue">
- <property name="maxSize" value="${droids.queue.maxSize}"/>
+ <bean id="java.util.LinkedList"
+ class="java.util.LinkedList">
</bean>
<!-- Protocol -->
<bean
Modified: incubator/droids/branch/bchapuis/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java (original)
+++ incubator/droids/branch/bchapuis/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java Wed Feb 2 22:29:28 2011
@@ -22,14 +22,10 @@ import java.util.concurrent.TimeUnit;
import junit.framework.Assert;
-import org.apache.droids.DroidsFactory;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Link;
-import org.apache.droids.api.TaskExceptionHandler;
-import org.apache.droids.api.TaskExceptionResult;
import org.apache.droids.examples.handler.ReportHandler;
import org.apache.droids.impl.ReportCrawlingDroid;
-import org.apache.droids.impl.SequentialTaskMaster;
import org.apache.droids.localserver.LocalHttpServer;
import org.apache.droids.localserver.ResourceHandler;
import org.apache.droids.robot.crawler.CrawlingDroid;
Modified: incubator/droids/branch/bchapuis/droids-spring/src/test/resources/droids-core-test-context.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-spring/src/test/resources/droids-core-test-context.xml?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-spring/src/test/resources/droids-core-test-context.xml (original)
+++ incubator/droids/branch/bchapuis/droids-spring/src/test/resources/droids-core-test-context.xml Wed Feb 2 22:29:28 2011
@@ -44,7 +44,7 @@
</bean>
<bean name="taskMaster"
- class="org.apache.droids.impl.SequentialTaskMaster">
+ class="org.apache.droids.impl.TaskMasterImpl">
<property name="exceptionHandler" ref="taskExceptionHandler" />
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
<!--<property name="maxThreads" value="${droids.maxThreads}"/>-->
@@ -53,17 +53,17 @@
<!-- Droids -->
<bean name="org.apache.droids.api.Droid/report"
class="org.apache.droids.impl.ReportCrawlingDroid">
+ <constructor-arg ref="java.util.LinkedList" />
<constructor-arg ref="taskMaster" />
- <constructor-arg ref="org.apache.droids.impl.SimpleTaskQueue" />
+
<property name="protocolFactory" ref="org.apache.droids.helper.factories.ProtocolFactory"/>
<property name="parserFactory" ref="org.apache.droids.helper.factories.ParserFactory"/>
<property name="filtersFactory" ref="org.apache.droids.helper.factories.URLFiltersFactory"/>
</bean>
<!-- Queue -->
- <bean id="org.apache.droids.impl.SimpleTaskQueue"
- class="org.apache.droids.impl.SimpleTaskQueue">
- <property name="maxSize" value="${droids.queue.maxSize}"/>
+ <bean id="java.util.LinkedList"
+ class="java.util.LinkedList">
</bean>
<!-- Protocol -->
<bean
Modified: incubator/droids/branch/bchapuis/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java (original)
+++ incubator/droids/branch/bchapuis/droids-wicket/src/main/java/org/apache/droids/wicket/component/DroidPanel.java Wed Feb 2 22:29:28 2011
@@ -17,17 +17,14 @@
package org.apache.droids.wicket.component;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
-import java.util.List;
+import java.util.Queue;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Task;
-import org.apache.droids.api.TaskQueue;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
+import org.apache.droids.impl.TaskMasterImpl;
import org.apache.droids.monitor.SimpleWorkMonitor;
import org.apache.droids.monitor.WorkBean;
import org.apache.wicket.ajax.AbstractAjaxTimerBehavior;
@@ -54,10 +51,10 @@ public class DroidPanel extends Panel
Droid droid;
@SpringBean( name="taskQueue" )
- TaskQueue taskQueue;
+ Queue taskQueue;
@SpringBean( name="taskMaster" )
- MultiThreadedTaskMaster taskMaster;
+ TaskMasterImpl taskMaster;
@SpringBean( name="runningMonitor" )
SimpleWorkMonitor monitor;
@@ -172,18 +169,7 @@ public class DroidPanel extends Panel
@Override
protected Iterator<IModel<Task>> getItemModels()
{
- Collection<Task> top = taskQueue.peek( 5 );
-
- List<IModel<Task>> models = new ArrayList<IModel<Task>>( top.size() );
- for( final Task m : top ) {
- models.add( new AbstractReadOnlyModel<Task>() {
- @Override
- public Task getObject() {
- return m;
- }
- });
- }
- return models.iterator();
+ return taskQueue.iterator();
}
@Override
@@ -197,7 +183,7 @@ public class DroidPanel extends Panel
queueDIV.add( new Label( "more", new AbstractReadOnlyModel<String>() {
@Override
public String getObject() {
- return taskQueue.getSize() + " tasks.";
+ return taskQueue.size() + " tasks.";
}
}));
queueDIV.setOutputMarkupId( true );
Modified: incubator/droids/branch/bchapuis/droids-wicket/src/main/resources/applicationContext.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branch/bchapuis/droids-wicket/src/main/resources/applicationContext.xml?rev=1066666&r1=1066665&r2=1066666&view=diff
==============================================================================
--- incubator/droids/branch/bchapuis/droids-wicket/src/main/resources/applicationContext.xml (original)
+++ incubator/droids/branch/bchapuis/droids-wicket/src/main/resources/applicationContext.xml Wed Feb 2 22:29:28 2011
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
@@ -17,34 +17,34 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<!DOCTYPE beans PUBLIC "-//SPRING//DTD BEAN//EN" "http://www.springframework.org/dtd/spring-beans.dtd">
-
+<!DOCTYPE beans PUBLIC "-//SPRING//DTD BEAN//EN" "http://www.springframework.org/dtd/spring-beans.dtd">
+
<beans>
<!-- use system variables -->
<bean id="placeholderConfig"
class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="location"><value>classpath:application.properties</value></property>
<property name="systemPropertiesModeName"><value>SYSTEM_PROPERTIES_MODE_OVERRIDE</value></property>
- </bean>
-
- <!-- setup wicket application -->
- <bean id="wicketApplication" class="org.apache.droids.wicket.app.DroidsApplication" />
-
-
- <bean id="runningMonitor" class="org.apache.droids.monitor.SimpleWorkMonitor" />
+ </bean>
+
+ <!-- setup wicket application -->
+ <bean id="wicketApplication" class="org.apache.droids.wicket.app.DroidsApplication" />
+
+
+ <bean id="runningMonitor" class="org.apache.droids.monitor.SimpleWorkMonitor" />
<!-- DROIDS -->
<bean name="taskMaster"
- class="org.apache.droids.impl.MultiThreadedTaskMaster">
+ class="org.apache.droids.impl.TaskMasterImpl">
<!--
<property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
-->
- <property name="maxThreads" value="1"/>
+ <property name="maxThreads" value="1"/>
<property name="monitor" ref="runningMonitor"/>
</bean>
<bean name="taskQueue"
- class="org.apache.droids.impl.SimpleTaskQueue">
+ class="java.util.concurrent.ConcurrentLinkedQueue">
<!-- class="org.apache.droids.impl.SimpleTaskQueue" -->
</bean>
@@ -54,4 +54,4 @@
<constructor-arg ref="taskMaster" />
</bean>
-</beans>
+</beans>