You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by to...@apache.org on 2012/12/14 09:30:27 UTC
svn commit: r1421765 - in /incubator/droids/branches/0.2.x-cleanup: ./
droids-core/src/main/java/org/apache/droids/core/
droids-core/src/main/java/org/apache/droids/handle/
droids-core/src/main/java/org/apache/droids/helper/factories/
droids-core/src/m...
Author: tobr
Date: Fri Dec 14 09:30:23 2012
New Revision: 1421765
URL: http://svn.apache.org/viewvc?rev=1421765&view=rev
Log:
added tests to show how easy it is to use droids
Added:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java (with props)
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java (with props)
incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml (with props)
Removed:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Parser.java
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/FilterFactory.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ParserFactory.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
incubator/droids/branches/0.2.x-cleanup/droids-walker/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/FileWorker.java
incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/SimpleWalkingDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/java/org/apache/droids/walker/WalkingDroidTest.java
incubator/droids/branches/0.2.x-cleanup/pom.xml
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java Fri Dec 14 09:30:23 2012
@@ -21,6 +21,9 @@ import org.apache.droids.helper.factorie
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.filter.Filter;
+import org.apache.droids.parse.Parser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Queue;
@@ -37,18 +40,19 @@ public abstract class AbstractDroid<T ex
protected FilterFactory<T> filterFactory;
protected HandlerFactory handlerFactory;
+ protected final static Logger logger = LoggerFactory.getLogger(AbstractDroid.class);
+
public AbstractDroid() {
- this.queue = new SimpleTaskQueueWithHistory<T>();
- this.taskMaster = new MultiThreadedTaskMaster<T>();
- this.protocolFactory = new ProtocolFactory();
- this.parserFactory = new ParserFactory();
- this.filterFactory = new FilterFactory();
- this.handlerFactory = new HandlerFactory();
+ this(new SimpleTaskQueueWithHistory<T>(), new MultiThreadedTaskMaster<T>());
}
public AbstractDroid(Queue<T> queue, TaskMaster<T> taskMaster) {
this.queue = queue;
this.taskMaster = taskMaster;
+ this.protocolFactory = new ProtocolFactory();
+ this.parserFactory = new ParserFactory();
+ this.filterFactory = new FilterFactory();
+ this.handlerFactory = new HandlerFactory();
}
@Override
@@ -81,24 +85,28 @@ public abstract class AbstractDroid<T ex
return this.filterFactory.filter(task);
}
+ @Override
public void addParsers(Parser... parsers) {
for (Parser parser : parsers) {
this.parserFactory.addParser(parser);
}
}
+ @Override
public void addProtocols(Protocol... protocols) {
for (Protocol protocol : protocols) {
this.protocolFactory.addProtocol(protocol);
}
}
+ @Override
public void addHandlers(Handler... handlers) {
for (Handler handler : handlers) {
this.handlerFactory.addHandler(handler);
}
}
+ @Override
public void addFilters(Filter<T>... filters) {
for (Filter<T> filter : filters) {
this.filterFactory.addFilter(filter);
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java Fri Dec 14 09:30:23 2012
@@ -27,5 +27,8 @@ public class ContentEntity {
this.data.put(key, value);
}
+ public Map<String, Object> getData() {
+ return data;
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java Fri Dec 14 09:30:23 2012
@@ -16,6 +16,9 @@
*/
package org.apache.droids.core;
+import org.apache.droids.filter.Filter;
+import org.apache.droids.parse.Parser;
+
import java.io.IOException;
/**
@@ -52,6 +55,14 @@ public interface Droid<T extends Task> {
*/
public Worker<T> getNewWorker();
+ public void addParsers(Parser... parsers);
+
+ public void addProtocols(Protocol... protocols);
+
+ public void addHandlers(Handler... handlers);
+
+ public void addFilters(Filter<T>... filters);
+
/**
* Parse the task
*
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java Fri Dec 14 09:30:23 2012
@@ -16,46 +16,44 @@
*/
package org.apache.droids.handle;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Reader;
-import java.io.Writer;
-
-import org.apache.droids.core.ContentEntity;
import org.apache.droids.core.DroidsException;
import org.apache.droids.core.Handler;
import org.apache.droids.core.Task;
-import org.apache.droids.exception.InvalidTaskException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
/**
* Handler that write the stream to the sysout. Mostly added for debugging
* reasons.
- *
+ *
* @version 1.0
- *
*/
public class SysoutHandler extends WriterHandler implements Handler {
+ private static final Logger logger = LoggerFactory.getLogger(SysoutHandler.class);
+
+ private static void writeOutput(InputStream stream) throws IOException {
+ Reader reader = new InputStreamReader(stream);
+ Writer output = new OutputStreamWriter(System.out);
+ pipe(reader, output);
+ }
- private static void writeOutput(InputStream stream) throws IOException {
- Reader reader = new InputStreamReader(stream);
- Writer output = new OutputStreamWriter(System.out);
- pipe(reader, output);
- }
-
- @Override
- public void handle(Task task) throws IOException, DroidsException {
- InputStream instream;
- if (task.getContentEntity().getValue(ContentEntity.CONTENT) instanceof InputStream) {
- instream = (InputStream)task.getContentEntity().getValue(ContentEntity.CONTENT);
- } else
- throw new InvalidTaskException("no inputstream available");
- try {
- writeOutput(instream);
- } finally {
- instream.close();
+ @Override
+ public void handle(Task task) throws IOException, DroidsException {
+ for (String key : task.getContentEntity().getData().keySet()) {
+ if (task.getContentEntity().getValue(key) instanceof InputStream) {
+ InputStream instream = (InputStream) task.getContentEntity().getValue(key);
+ try {
+ logger.info(key + ": ");
+ writeOutput(instream);
+ } finally {
+ instream.close();
+ }
+ } else {
+ logger.info(key + ": " + task.getContentEntity().getValue(key).toString());
+ }
+ }
}
- }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/FilterFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/FilterFactory.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/FilterFactory.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/FilterFactory.java Fri Dec 14 09:30:23 2012
@@ -19,6 +19,7 @@ package org.apache.droids.helper.factori
import org.apache.droids.core.Task;
import org.apache.droids.filter.Filter;
+import java.util.HashSet;
import java.util.Set;
/**
@@ -26,16 +27,20 @@ import java.util.Set;
*
* @version 1.0
*/
-public class FilterFactory<T extends Task> {
+public class FilterFactory {
private Set<Filter> filters;
+ public FilterFactory() {
+ this.filters = new HashSet<Filter>();
+ }
+
public <T extends Task> void addFilter(Filter<T> filter) {
filters.add(filter);
}
public <T extends Task> T filter(T task) {
for (Filter<T> filter : filters) {
- // task =
+ task = filter.filter(task);
}
return task;
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java Fri Dec 14 09:30:23 2012
@@ -17,6 +17,7 @@
package org.apache.droids.helper.factories;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Set;
import org.apache.droids.core.DroidsException;
@@ -32,6 +33,10 @@ import org.apache.droids.core.Task;
public class HandlerFactory {
private Set<Handler> handlers;
+ public HandlerFactory() {
+ this.handlers = new HashSet<Handler>();
+ }
+
public void addHandler(Handler handler) {
handlers.add(handler);
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ParserFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ParserFactory.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ParserFactory.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ParserFactory.java Fri Dec 14 09:30:23 2012
@@ -17,10 +17,11 @@
package org.apache.droids.helper.factories;
import org.apache.droids.core.DroidsException;
-import org.apache.droids.core.Parser;
+import org.apache.droids.parse.Parser;
import org.apache.droids.core.Task;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Set;
/**
@@ -31,6 +32,10 @@ import java.util.Set;
public class ParserFactory {
private Set<Parser> parsers;
+ public ParserFactory() {
+ this.parsers = new HashSet<Parser>();
+ }
+
public void addParser(Parser parser) {
this.parsers.add(parser);
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java Fri Dec 14 09:30:23 2012
@@ -20,6 +20,7 @@ import org.apache.droids.core.Protocol;
import org.apache.droids.exception.ProtocolNotFoundException;
import java.net.URI;
+import java.util.HashSet;
import java.util.Set;
/**
@@ -30,6 +31,10 @@ import java.util.Set;
public class ProtocolFactory {
private Set<Protocol> protocols;
+ public ProtocolFactory() {
+ this.protocols = new HashSet<Protocol>();
+ }
+
public void addProtocol(Protocol protocol) {
protocols.add(protocol);
}
Added: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java?rev=1421765&view=auto
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java (added)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java Fri Dec 14 09:30:23 2012
@@ -0,0 +1,23 @@
+package org.apache.droids.parse;
+
+import org.apache.droids.core.DroidsException;
+import org.apache.droids.core.Task;
+
+import java.io.IOException;
+
+/**
+ *
+ *
+ *
+ */
+public class FileNameParser implements Parser {
+
+ public static final String FILENAME = "filename";
+
+
+ @Override
+ public void parse(Task task) throws DroidsException, IOException {
+ String path = task.getURI().getPath();
+ task.getContentEntity().put(FILENAME, path.substring(path.lastIndexOf('/') + 1));
+ }
+}
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/FileNameParser.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java?rev=1421765&view=auto
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java (added)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java Fri Dec 14 09:30:23 2012
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.parse;
+
+import java.io.IOException;
+
+import org.apache.droids.core.DroidsException;
+import org.apache.droids.core.Task;
+
+/**
+ * Simple parser that is only forcing to return a parse object.
+ *
+ * @version 1.0
+ *
+ */
+public interface Parser {
+ /**
+ * Creates the parse for some content.
+ *
+ * @param task
+ * the task that correspond to the stream
+ * @return the parse object
+ */
+ public void parse(Task task) throws DroidsException, IOException;
+}
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/parse/Parser.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java Fri Dec 14 09:30:23 2012
@@ -29,7 +29,7 @@ import org.apache.droids.core.Worker;
import org.apache.droids.helper.factories.HandlerFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
import org.apache.droids.parse.Parse;
-import org.apache.droids.core.Parser;
+import org.apache.droids.parse.Parser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Modified: incubator/droids/branches/0.2.x-cleanup/droids-walker/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-walker/pom.xml?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-walker/pom.xml (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-walker/pom.xml Fri Dec 14 09:30:23 2012
@@ -11,7 +11,7 @@
</parent>
<artifactId>droids-walker</artifactId>
- <name>droids-walker</name>
+ <name>APACHE DROIDS WALKER</name>
<dependencies>
<dependency>
@@ -28,7 +28,7 @@
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
- <version>1.0.7</version>
+ <version>${logback.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
Modified: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/FileWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/FileWorker.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/FileWorker.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/FileWorker.java Fri Dec 14 09:30:23 2012
@@ -16,36 +16,41 @@
*/
package org.apache.droids.walker;
-import java.io.File;
-
+import org.apache.droids.core.DroidsException;
import org.apache.droids.core.Worker;
-import org.apache.droids.exception.InvalidTaskException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+
public class FileWorker implements Worker<FileTask> {
- private final Logger logger = LoggerFactory.getLogger(FileWorker.class);
- final WalkingDroid droid;
+ private final Logger logger = LoggerFactory.getLogger(FileWorker.class);
+ final WalkingDroid droid;
- public FileWorker(WalkingDroid droid) {
- this.droid = droid;
- }
-
- @Override
- public void execute(FileTask task) throws InvalidTaskException {
- File file = task.getFile();
- if (file.isDirectory()) {
- File[] files = file.listFiles();
- if (files != null) {
- for (File f : files) {
- droid.add(new FileTask(f, task.getDepth() + 1));
- }
- }
- } else {
- if (logger.isInfoEnabled()) {
- logger.info("FILE: " + file.getAbsolutePath());
- }
- }
- }
+ public FileWorker(WalkingDroid droid) {
+ this.droid = droid;
+ }
+
+ @Override
+ public void execute(FileTask task) throws IOException, DroidsException {
+ File file = task.getFile();
+ if (file.isDirectory()) {
+ File[] files = file.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ FileTask newTask = droid.filter(new FileTask(f, task.getDepth() + 1));
+ if (newTask != null)
+ droid.add(newTask);
+ }
+ }
+ } else {
+ if (logger.isDebugEnabled()) {
+ logger.debug("FILE: " + file.getAbsolutePath());
+ }
+ droid.parse(task);
+ droid.handle(task);
+ }
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/SimpleWalkingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/SimpleWalkingDroid.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/SimpleWalkingDroid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-walker/src/main/java/org/apache/droids/walker/SimpleWalkingDroid.java Fri Dec 14 09:30:23 2012
@@ -16,54 +16,52 @@
*/
package org.apache.droids.walker;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.concurrent.TimeUnit;
-
import com.google.common.base.Preconditions;
-import java.util.LinkedList;
-import java.util.Queue;
-
import org.apache.droids.core.AbstractDroid;
-import org.apache.droids.core.MultiThreadedTaskMaster;
import org.apache.droids.core.TaskMaster;
-import org.apache.droids.exception.InvalidTaskException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.util.Collection;
+import java.util.Queue;
+
public class SimpleWalkingDroid extends AbstractDroid<FileTask> implements WalkingDroid {
private Collection<File> initialFiles;
private static Logger logger = LoggerFactory.getLogger(SimpleWalkingDroid.class);
- public SimpleWalkingDroid(Queue<FileTask> queue, TaskMaster<FileTask> taskMaster) {
- super(queue, taskMaster);
- }
-
- @Override
- public void setInitialFiles(Collection<File> initialFiles) {
- this.initialFiles = initialFiles;
- }
-
- @Override
- public void start() {
- Preconditions.checkState(initialFiles != null,
- "FileSystemWalker requires at least one starting file");
- Preconditions.checkState(!initialFiles.isEmpty(),
- "FileSystemWalker requires at least one starting file");
- for (File file : initialFiles) {
- queue.add(new FileTask(file, 0));
- }
+ public SimpleWalkingDroid() {
+
+ }
+
+ public SimpleWalkingDroid(Queue<FileTask> queue, TaskMaster<FileTask> taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ @Override
+ public void setInitialFiles(Collection<File> initialFiles) {
+ this.initialFiles = initialFiles;
+ }
+
+ @Override
+ public void start() {
+ Preconditions.checkState(initialFiles != null,
+ "FileSystemWalker requires at least one starting file");
+ Preconditions.checkState(!initialFiles.isEmpty(),
+ "FileSystemWalker requires at least one starting file");
+ for (File file : initialFiles) {
+ queue.add(new FileTask(file, 0));
+ }
super.start();
- }
+ }
- @Override
- public void finished() {
- logger.info("FINISHED!!!");
- }
-
- @Override
- public FileWorker getNewWorker() {
- return new FileWorker(this);
- }
+ @Override
+ public void finished() {
+ logger.info("FINISHED!!!");
+ }
+
+ @Override
+ public FileWorker getNewWorker() {
+ return new FileWorker(this);
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/java/org/apache/droids/walker/WalkingDroidTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/java/org/apache/droids/walker/WalkingDroidTest.java?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/java/org/apache/droids/walker/WalkingDroidTest.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/java/org/apache/droids/walker/WalkingDroidTest.java Fri Dec 14 09:30:23 2012
@@ -1,32 +1,61 @@
package org.apache.droids.walker;
+import org.apache.droids.core.SequentialTaskMaster;
+import org.apache.droids.core.TaskMaster;
+import org.apache.droids.filter.RegexURLFilter;
+import org.apache.droids.handle.SysoutHandler;
+import org.apache.droids.parse.FileNameParser;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
import java.io.File;
import java.util.Collection;
import java.util.LinkedList;
import java.util.Queue;
-import org.apache.droids.core.SequentialTaskMaster;
-import org.apache.droids.core.TaskMaster;
-import org.apache.droids.exception.InvalidTaskException;
-import org.junit.Test;
-
public class WalkingDroidTest {
@Test
- public void testWalkingDroid() throws InvalidTaskException {
+ public void testWalkingDroid() throws NullPointerException {
Queue<FileTask> queue = new LinkedList<FileTask>();
TaskMaster<FileTask> taskMaster = new SequentialTaskMaster<FileTask>();
-
+
Collection<File> initialFiles = new LinkedList<File>();
- try {
- initialFiles.add(new File(this.getClass().getClassLoader().getResource("docs").getFile()));
- } catch (NullPointerException npe) {
- npe.printStackTrace();
- }
+
+ initialFiles.add(new File(this.getClass().getClassLoader().getResource("docs").getFile()));
+
SimpleWalkingDroid droid = new SimpleWalkingDroid(queue, taskMaster);
droid.setInitialFiles(initialFiles);
-
+ droid.addParsers(new FileNameParser());
+ droid.addHandlers(new SysoutHandler());
+
droid.start();
-
- }
+
+ assertEquals(19, taskMaster.getCompletedTasks());
+ }
+
+ @Test
+ public void testFilteredWalkingDroid() {
+ Queue<FileTask> queue = new LinkedList<FileTask>();
+ TaskMaster<FileTask> taskMaster = new SequentialTaskMaster<FileTask>();
+
+ Collection<File> initialFiles = new LinkedList<File>();
+
+ initialFiles.add(new File(this.getClass().getClassLoader().getResource("docs").getFile()));
+
+ SimpleWalkingDroid droid = new SimpleWalkingDroid(queue, taskMaster);
+ droid.setInitialFiles(initialFiles);
+ droid.addParsers(new FileNameParser());
+ droid.addHandlers(new SysoutHandler());
+
+ RegexURLFilter filter = new RegexURLFilter();
+ filter.addRule(false, ".*(\\.pdf)");
+ filter.addRule(true, ".*");
+ droid.addFilters(filter);
+ droid.start();
+
+ assertEquals(10, taskMaster.getCompletedTasks());
+
+
+ }
}
Added: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml?rev=1421765&view=auto
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml (added)
+++ incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml Fri Dec 14 09:30:23 2012
@@ -0,0 +1,17 @@
+<configuration>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <!-- encoders are assigned the type
+ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="org.apache.droids" level="info"/>
+
+
+ <root level="error">
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>
\ No newline at end of file
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Propchange: incubator/droids/branches/0.2.x-cleanup/droids-walker/src/test/resources/logback.xml
------------------------------------------------------------------------------
svn:mime-type = text/xml
Modified: incubator/droids/branches/0.2.x-cleanup/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/pom.xml?rev=1421765&r1=1421764&r2=1421765&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/pom.xml (original)
+++ incubator/droids/branches/0.2.x-cleanup/pom.xml Fri Dec 14 09:30:23 2012
@@ -183,6 +183,7 @@
<httpclient.version>4.2</httpclient.version>
<nekohtml.version>1.9.15</nekohtml.version>
<slf4j.version>1.6.4</slf4j.version>
+ <logback.version>1.0.7</logback.version>
<junit.version>4.10</junit.version>
</properties>