You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by to...@apache.org on 2012/09/10 13:03:49 UTC
svn commit: r1382814 - in /incubator/droids/trunk:
droids-core/src/main/java/org/apache/droids/impl/
droids-core/src/main/java/org/apache/droids/robot/crawler/
droids-core/src/test/java/org/apache/droids/robot/crawler/ droids-examples/
droids-examples/...
Author: tobr
Date: Mon Sep 10 13:03:48 2012
New Revision: 1382814
URL: http://svn.apache.org/viewvc?rev=1382814&view=rev
Log:
updated examples
DROIDS-89
Added:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/ReportCrawlingDroid.java (props changed)
- copied unchanged from r1382765, incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/ReportCrawlingDroid.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java (contents, props changed)
- copied, changed from r1382765, incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/SimpleRuntime.java
Removed:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/robot/crawler/ReportCrawlingDroid.java
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/droids-example-context.xml
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/SimpleRuntime.java
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java
incubator/droids/trunk/droids-examples/pom.xml
incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
incubator/droids/trunk/src/site/apt/getting-started.apt
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java?rev=1382814&r1=1382813&r2=1382814&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/impl/SequentialTaskMaster.java Mon Sep 10 13:03:48 2012
@@ -89,10 +89,13 @@ public class SequentialTaskMaster<T exte
}
switch (result) {
case WARN:
- LOG.warn(ex.toString(), ex);
+ LOG.warn(ex.toString() + " " + task.getId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ex.toString(), ex);
+ }
break;
case FATAL:
- LOG.warn(ex.getMessage(), ex);
+ LOG.error(ex.getMessage(), ex);
terminated = true;
break;
}
Propchange: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/ReportCrawlingDroid.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/ReportCrawlingDroid.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Propchange: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/ReportCrawlingDroid.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/droids/trunk/droids-examples/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/pom.xml?rev=1382814&r1=1382813&r2=1382814&view=diff
==============================================================================
--- incubator/droids/trunk/droids-examples/pom.xml (original)
+++ incubator/droids/trunk/droids-examples/pom.xml Mon Sep 10 13:03:48 2012
@@ -48,6 +48,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-tika</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
@@ -69,7 +74,6 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.6.4</version>
- <scope>test</scope>
</dependency>
</dependencies>
Modified: incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java?rev=1382814&r1=1382813&r2=1382814&view=diff
==============================================================================
--- incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java (original)
+++ incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java Mon Sep 10 13:03:48 2012
@@ -20,7 +20,6 @@ package org.apache.droids.examples;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.Queue;
@@ -37,10 +36,10 @@ import org.apache.droids.helper.factorie
import org.apache.droids.impl.DefaultTaskExceptionHandler;
import org.apache.droids.impl.SequentialTaskMaster;
import org.apache.droids.impl.SimpleTaskQueueWithHistory;
-import org.apache.droids.parse.html.HtmlParser;
import org.apache.droids.protocol.http.HttpProtocol;
import org.apache.droids.robot.crawler.CrawlingDroid;
import org.apache.droids.robot.crawler.ReportCrawlingDroid;
+import org.apache.droids.tika.TikaDocumentParser;
/**
@@ -53,13 +52,8 @@ public class DroidsFactory
public static ParserFactory createDefaultParserFactory() {
ParserFactory parserFactory = new ParserFactory();
- HtmlParser htmlParser = new HtmlParser();
- htmlParser.setElements(new HashMap<String, String>());
- htmlParser.getElements().put("a", "href");
- htmlParser.getElements().put("link", "href");
- htmlParser.getElements().put("img", "src");
- htmlParser.getElements().put("script", "src");
- parserFactory.getMap().put("text/html", htmlParser);
+ TikaDocumentParser tikaParser = new TikaDocumentParser();
+ parserFactory.getMap().put("text/html", tikaParser);
return parserFactory;
}
Copied: incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java (from r1382765, incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/SimpleRuntime.java)
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java?p2=incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java&p1=incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/SimpleRuntime.java&r1=1382765&r2=1382814&rev=1382814&view=diff
==============================================================================
--- incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java Mon Sep 10 13:03:48 2012
@@ -14,18 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.droids.examples;
+package org.apache.droids.examples.cli;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.TimeUnit;
import org.apache.droids.robot.crawler.CrawlingDroid;
+import org.apache.droids.tika.TikaDocumentParser;
import org.apache.droids.api.Link;
import org.apache.droids.delay.SimpleDelayTimer;
+import org.apache.droids.examples.SysoutCrawlingDroid;
import org.apache.droids.handle.SysoutHandler;
import org.apache.droids.helper.factories.DroidFactory;
import org.apache.droids.helper.factories.HandlerFactory;
@@ -35,7 +36,6 @@ import org.apache.droids.helper.factorie
import org.apache.droids.impl.DefaultTaskExceptionHandler;
import org.apache.droids.impl.SequentialTaskMaster;
import org.apache.droids.net.RegexURLFilter;
-import org.apache.droids.parse.html.HtmlParser;
import org.apache.droids.protocol.http.DroidsHttpClient;
import org.apache.droids.protocol.http.HttpProtocol;
import org.apache.http.HttpVersion;
@@ -67,13 +67,8 @@ public class SimpleRuntime {
// Create parser factory. Support basic HTML markup only
ParserFactory parserFactory = new ParserFactory();
- HtmlParser htmlParser = new HtmlParser();
- htmlParser.setElements(new HashMap<String, String>());
- htmlParser.getElements().put("a", "href");
- htmlParser.getElements().put("link", "href");
- htmlParser.getElements().put("img", "src");
- htmlParser.getElements().put("script", "src");
- parserFactory.getMap().put("text/html", htmlParser);
+ TikaDocumentParser tikaParser = new TikaDocumentParser();
+ parserFactory.getMap().put("text/html", tikaParser);
// Create protocol factory. Support HTTP/S only.
ProtocolFactory protocolFactory = new ProtocolFactory();
Propchange: incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Propchange: incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
------------------------------------------------------------------------------
svn:mergeinfo =
Propchange: incubator/droids/trunk/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java?rev=1382814&r1=1382813&r2=1382814&view=diff
==============================================================================
--- incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java (original)
+++ incubator/droids/trunk/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java Mon Sep 10 13:03:48 2012
@@ -20,7 +20,6 @@ package org.apache.droids.examples;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.Queue;
@@ -32,9 +31,9 @@ import org.apache.droids.examples.Sysout
import org.apache.droids.exception.DroidsException;
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
-import org.apache.droids.parse.html.HtmlParser;
import org.apache.droids.robot.crawler.CrawlingDroid;
import org.apache.droids.robot.crawler.CrawlingWorker;
+import org.apache.droids.tika.TikaDocumentParser;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -44,7 +43,7 @@ import org.mockito.internal.stubbing.def
public class TestCrawlingWorker {
CrawlingWorker instance;
- private HtmlParser htmlParser;
+ private TikaDocumentParser htmlParser;
@Before
public void initialize() {
@@ -66,9 +65,8 @@ public class TestCrawlingWorker {
private final ParserFactory parserSetup() {
final ParserFactory parserFactory = new ParserFactory();
- final HashMap<String, Object> parserFactoryMap = new HashMap<String, Object>();
- htmlParser = Mockito.mock(HtmlParser.class, new ReturnsMocks());
+ htmlParser = Mockito.mock(TikaDocumentParser.class, new ReturnsMocks());
parserFactory.getMap().put("text/html", htmlParser);
return parserFactory;
Modified: incubator/droids/trunk/src/site/apt/getting-started.apt
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/site/apt/getting-started.apt?rev=1382814&r1=1382813&r2=1382814&view=diff
==============================================================================
--- incubator/droids/trunk/src/site/apt/getting-started.apt (original)
+++ incubator/droids/trunk/src/site/apt/getting-started.apt Mon Sep 10 13:03:48 2012
@@ -51,6 +51,6 @@ Getting Started
More Examples
- More examples are available in the test cases.
- Here you can see the way to work with Droids.
- Our test cases demonstrate best practice code and if you want to develop your own droids you should look at them.
\ No newline at end of file
+ More examples are available in the droids-examples module.
+ Here you can see how to work with Droids.
+ These test cases demonstrate best practice code and if you want to develop your own droids you should look at them.
\ No newline at end of file