You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by th...@apache.org on 2008/03/11 14:27:01 UTC
svn commit: r635918 - in /labs/droids/trunk/src:
core/java/org/apache/droids/ core/java/org/apache/droids/handle/ examples/
examples/java/ examples/java/org/ examples/java/org/apache/
examples/java/org/apache/droids/ examples/java/org/apache/droids/exa...
Author: thorsten
Date: Tue Mar 11 06:27:00 2008
New Revision: 635918
URL: http://svn.apache.org/viewvc?rev=635918&view=rev
Log:
Starting a simple indexer for solr as example.
Added:
labs/droids/trunk/src/core/java/org/apache/droids/handle/Solr.java (with props)
labs/droids/trunk/src/examples/
labs/droids/trunk/src/examples/java/
labs/droids/trunk/src/examples/java/org/
labs/droids/trunk/src/examples/java/org/apache/
labs/droids/trunk/src/examples/java/org/apache/droids/
labs/droids/trunk/src/examples/java/org/apache/droids/examples/
labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java (with props)
labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java (with props)
labs/droids/trunk/src/examples/java/org/apache/droids/examples/droids-example-context.xml (with props)
Modified:
labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java?rev=635918&r1=635917&r2=635918&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java Tue Mar 11 06:27:00 2008
@@ -52,11 +52,11 @@
final String userAgent = this.getClass().getCanonicalName();
Core.threadMessage("Starting " + userAgent);
try {
- uri = link.getId();
- Core.threadMessage("uri " + uri);
- protocol = protocolFactory.getProtocol(uri);
- if (protocol.isAllowed(uri)) {
- String contentType = protocol.getContentType(uri);
+ setUri(link.getId());
+ Core.threadMessage("uri " + getUri());
+ setProtocol(protocolFactory.getProtocol(getUri()));
+ if (getProtocol().isAllowed(getUri())) {
+ String contentType = getProtocol().getContentType(getUri());
Core.threadMessage("contentType " + contentType);
parser = parserFactory.getParser(contentType);
// parse contains the outlinks and can be used later
@@ -77,7 +77,7 @@
protected void handle(Parse parse) throws MalformedURLException, IOException {
if (null != parse)
- handlerFactory.handle(protocol.openStream(uri), new URL(uri), parse);
+ handlerFactory.handle(getProtocol().openStream(getUri()), new URL(getUri()), parse);
}
protected Parse getParse() {
@@ -85,7 +85,7 @@
if (null != parser) {
try {
// extract links
- parse = parser.getParse(protocol.openStream(uri), link);
+ parse = parser.getParse(getProtocol().openStream(getUri()), link);
// all links from the page unfiltered
filter(parse);
} catch (Exception e) {
@@ -165,6 +165,22 @@
public synchronized Droid getDroid() {
return droid;
+ }
+
+ public synchronized void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ public synchronized String getUri() {
+ return uri;
+ }
+
+ public synchronized void setProtocol(Protocol protocol) {
+ this.protocol = protocol;
+ }
+
+ public synchronized Protocol getProtocol() {
+ return protocol;
}
}
Added: labs/droids/trunk/src/core/java/org/apache/droids/handle/Solr.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/handle/Solr.java?rev=635918&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/handle/Solr.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/handle/Solr.java Tue Mar 11 06:27:00 2008
@@ -0,0 +1,17 @@
+package org.apache.droids.handle;
+
+import java.io.InputStream;
+import java.net.URL;
+
+import org.apache.droids.api.Handler;
+import org.apache.droids.api.Parse;
+
+public class Solr implements Handler{
+
+ public void handle(InputStream openStream, URL url, Parse parse)
+ throws Exception {
+
+
+ }
+
+}
Propchange: labs/droids/trunk/src/core/java/org/apache/droids/handle/Solr.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java?rev=635918&view=auto
==============================================================================
--- labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java (added)
+++ labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java Tue Mar 11 06:27:00 2008
@@ -0,0 +1,13 @@
+package org.apache.droids.examples;
+
+import org.apache.droids.DefaultCrawler;
+import org.apache.droids.api.Worker;
+
+public class IndexerCrawler extends DefaultCrawler{
+ public synchronized Worker getWorker() {
+ IndexerWorker worker = new IndexerWorker();
+ worker.setQueue(getQueue());
+ worker.setDroid(this);
+ return worker;
+ }
+}
Propchange: labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java?rev=635918&view=auto
==============================================================================
--- labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java (added)
+++ labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java Tue Mar 11 06:27:00 2008
@@ -0,0 +1,22 @@
+package org.apache.droids.examples;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import org.apache.droids.Core;
+import org.apache.droids.DefaultWorker;
+import org.apache.droids.api.Handler;
+import org.apache.droids.api.Parse;
+
+public class IndexerWorker extends DefaultWorker {
+ protected void handle(Parse parse) throws MalformedURLException, IOException {
+ Handler handler = (Handler) getHandlerFactory().resolve("solr");
+ try {
+ handler.handle(getProtocol().openStream(getUri()), new URL(getUri()), parse);
+ } catch (Exception e) {
+ Core.threadMessage(e.getMessage());
+ }
+ getDroid().finishedWorker(super.getId());
+ }
+}
Propchange: labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: labs/droids/trunk/src/examples/java/org/apache/droids/examples/droids-example-context.xml
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/examples/java/org/apache/droids/examples/droids-example-context.xml?rev=635918&view=auto
==============================================================================
--- labs/droids/trunk/src/examples/java/org/apache/droids/examples/droids-example-context.xml (added)
+++ labs/droids/trunk/src/examples/java/org/apache/droids/examples/droids-example-context.xml Tue Mar 11 06:27:00 2008
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE beans PUBLIC "-//SPRING//DTD BEAN 2.0//EN"
+ "http://www.springframework.org/dtd/spring-beans-2.0.dtd">
+<beans>
+ <import
+ resource="classpath:/org/apache/droids/droids-core-factories-context.xml"/>
+ <import resource="classpath:/org/apache/droids/droids-core-context.xml"/>
+ <bean id="org.apache.droids.helper.factories.DroidFactory"
+ class="org.apache.droids.helper.factories.DroidFactory">
+ <property name="map">
+ <map>
+ <entry key="indexer" value-ref="indexer"/>
+ <!-- Referencing as well the default crawler
+ that is configured by the Droids core -->
+ <entry key="default" value-ref="default"/>
+ </map>
+ </property>
+ </bean>
+
+ <!-- Indexer -->
+ <bean id="indexer" class="org.apache.droids.examples.IndexerCrawler">
+ <property name="core" ref="org.apache.droids.Core"/>
+ <property name="queue" ref="org.apache.droids.queue.Simple"/>
+ <property name="maxThreads" value="@droids.maxThreads@"/>
+ <property name="url" value="@droids.initial.url@"/>
+ </bean>
+
+ <bean id="org.apache.droids.helper.factories.HandlerFactory"
+ class="org.apache.droids.helper.factories.HandlerFactory">
+ <property name="map">
+ <map>
+ <entry key="solr" value-ref="org.apache.droids.handle.Save"/>
+ </map>
+ </property>
+ </bean>
+
+ <!-- Handler -->
+ <bean id="org.apache.droids.handle.Solr" class="org.apache.droids.handle.Solr">
+ <property name="urpdateUrl" value="http://localhost:8983/solr/update"/>
+ </bean>
+</beans>
\ No newline at end of file
Propchange: labs/droids/trunk/src/examples/java/org/apache/droids/examples/droids-example-context.xml
------------------------------------------------------------------------------
svn:eol-style = native
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org