You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by ol...@apache.org on 2008/11/15 22:25:46 UTC
svn commit: r717938 - in /incubator/droids/trunk/droids-core/src:
main/java/org/apache/droids/api/ main/java/org/apache/droids/handle/
main/java/org/apache/droids/helper/factories/
main/java/org/apache/droids/protocol/file/ main/java/org/apache/droids/...
Author: olegk
Date: Sat Nov 15 14:25:45 2008
New Revision: 717938
URL: http://svn.apache.org/viewvc?rev=717938&view=rev
Log:
ContentEntity extended to contain the result of the parser stage
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ContentEntity.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Handler.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ManagedContentEntity.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Save.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Sysout.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/file/FileProtocol.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ContentEntity.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ContentEntity.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ContentEntity.java Sat Nov 15 14:25:45 2008
@@ -54,10 +54,18 @@
/**
* Returns charset of the entity if known. Otherwise returns
- * <code>null</null>.
+ * <code>null</code>.
*
* @return charset
*/
String getCharset();
+
+ /**
+ * Returns the parse object from a former processing step
+ * May be <code>null</code> if not available.
+ *
+ * @return parse object
+ */
+ Parse getParse();
}
\ No newline at end of file
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Handler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Handler.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Handler.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/Handler.java Sat Nov 15 14:25:45 2008
@@ -17,7 +17,6 @@
package org.apache.droids.api;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import org.apache.droids.exception.DroidsException;
@@ -35,10 +34,8 @@
* the underlying stream
* @param uri
* the uri we are currently processing
- * @param parse
- * the parse object from a former processing step
* @throws Exception
*/
- void handle(InputStream openStream, URI uri, Parse parse)
+ void handle(URI uri, ContentEntity entity)
throws IOException, DroidsException;
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ManagedContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ManagedContentEntity.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ManagedContentEntity.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/api/ManagedContentEntity.java Sat Nov 15 14:25:45 2008
@@ -27,7 +27,14 @@
public interface ManagedContentEntity extends ContentEntity {
/**
- * Release all underlying resources held by the entity.
+ * Sets the parse object.
+ *
+ * @param
+ */
+ void setParse(Parse parse);
+
+ /**
+ * Releases all underlying resources held by the entity.
*/
void finish();
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Save.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Save.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Save.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Save.java Sat Nov 15 14:25:45 2008
@@ -24,8 +24,8 @@
import java.io.OutputStream;
import java.net.URI;
+import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Handler;
-import org.apache.droids.api.Parse;
/**
* Handler which is writing the stream to the file system.
@@ -45,16 +45,15 @@
private boolean includeHost = false;
protected int bufferSize=8192;
- /*
- * (non-Javadoc)
- *
- * @see org.apache.droids.api.Handler#handle(java.io.InputStream,
- * java.net.URL, org.apache.droids.api.Parse)
- */
- public void handle(InputStream stream, URI urlToHandle, Parse parse)
- throws IOException {
- this.uri = urlToHandle;
- writeOutput(stream);
+
+ public void handle(URI uri, ContentEntity entity) throws IOException {
+ this.uri = uri;
+ InputStream instream = entity.obtainContent();
+ try {
+ writeOutput(instream);
+ } finally {
+ instream.close();
+ }
}
private void writeOutput(InputStream stream) throws IOException {
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Sysout.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Sysout.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Sysout.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/handle/Sysout.java Sat Nov 15 14:25:45 2008
@@ -24,8 +24,8 @@
import java.io.Writer;
import java.net.URI;
+import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Handler;
-import org.apache.droids.api.Parse;
/**
* Handler that write the stream to the sysout. Mostly added for debugging
@@ -42,14 +42,13 @@
pipe(reader, output);
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.droids.api.Handler#handle(java.io.InputStream,
- * java.net.URL, org.apache.droids.api.Parse)
- */
- public void handle(InputStream stream, URI uri, Parse parse) throws IOException {
- writeOutput(stream);
+ public void handle(URI uri, ContentEntity entity) throws IOException {
+ InputStream instream = entity.obtainContent();
+ try {
+ writeOutput(instream);
+ } finally {
+ instream.close();
+ }
}
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/helper/factories/HandlerFactory.java Sat Nov 15 14:25:45 2008
@@ -17,12 +17,10 @@
package org.apache.droids.helper.factories;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Handler;
-import org.apache.droids.api.Parse;
import org.apache.droids.exception.DroidsException;
/**
@@ -45,15 +43,10 @@
* the underlying parse object
* @return false if we found a problem, true if all went well
*/
- public boolean handle(ContentEntity entity, URI uri, Parse parse)
+ public boolean handle(URI uri, ContentEntity entity)
throws DroidsException, IOException {
for (Handler handler : getMap().values()) {
- InputStream instream = entity.obtainContent();
- try {
- handler.handle(instream, uri, parse);
- } finally {
- instream.close();
- }
+ handler.handle(uri, entity);
}
return true;
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/file/FileProtocol.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/file/FileProtocol.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/file/FileProtocol.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/file/FileProtocol.java Sat Nov 15 14:25:45 2008
@@ -8,6 +8,7 @@
import java.net.URI;
import org.apache.droids.api.ManagedContentEntity;
+import org.apache.droids.api.Parse;
import org.apache.droids.api.Protocol;
public class FileProtocol implements Protocol {
@@ -38,6 +39,8 @@
private final File file;
private final String mimeType;
private final String charset;
+
+ private Parse parse = null;
public FileContentEntity(File file) throws IOException {
super();
@@ -70,6 +73,14 @@
return charset;
}
+ public Parse getParse() {
+ return this.parse;
+ }
+
+ public void setParse(Parse parse) {
+ this.parse = parse;
+ }
+
}
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java Sat Nov 15 14:25:45 2008
@@ -21,6 +21,7 @@
import java.util.Locale;
import org.apache.droids.api.ManagedContentEntity;
+import org.apache.droids.api.Parse;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
@@ -34,6 +35,8 @@
private final String mimeType;
private final String charset;
+ private Parse parse = null;
+
public HttpContentEntity(HttpEntity entity) throws IOException {
super();
if (entity.isRepeatable()) {
@@ -84,6 +87,14 @@
return entity.getContent();
}
+ public Parse getParse() {
+ return this.parse;
+ }
+
+ public void setParse(Parse parse) {
+ this.parse = parse;
+ }
+
public void finish() {
}
Modified: incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java Sat Nov 15 14:25:45 2008
@@ -76,7 +76,8 @@
Collection<Link> outlinks = getFilteredOutlinks( parse );
droid.getQueue().merge( outlinks );
}
- handle( parse, entity, link );
+ entity.setParse(parse);
+ handle(entity, link);
}
}
} finally {
@@ -89,10 +90,10 @@
}
}
- protected void handle( Parse parse, ContentEntity entity, Link link )
+ protected void handle(ContentEntity entity, Link link)
throws DroidsException, IOException
{
- droid.getHandlerFactory().handle(entity, link.getURI(), parse);
+ droid.getHandlerFactory().handle(link.getURI(), entity);
}
protected Collection<Link> getFilteredOutlinks( Parse parse )
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java Sat Nov 15 14:25:45 2008
@@ -19,7 +19,6 @@
import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
-import org.apache.droids.api.Parse;
import org.apache.droids.robot.crawler.CrawlingWorker;
public class IndexerWorker extends CrawlingWorker {
@@ -34,7 +33,7 @@
@Override
- protected void handle( Parse parse, ContentEntity entity, Link link )
+ protected void handle(ContentEntity entity, Link link)
{
// TODO -- something different...
}
Modified: incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java?rev=717938&r1=717937&r2=717938&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java (original)
+++ incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java Sat Nov 15 14:25:45 2008
@@ -17,7 +17,6 @@
package org.apache.droids.impl;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -26,10 +25,10 @@
import junit.framework.Assert;
import org.apache.droids.DroidsFactory;
+import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Droid;
import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
-import org.apache.droids.api.Parse;
import org.apache.droids.api.TaskExceptionHandler;
import org.apache.droids.api.TaskExceptionResult;
import org.apache.droids.localserver.LocalHttpServer;
@@ -66,7 +65,7 @@
Handler testHandler = new Handler() {
- public void handle(InputStream openStream, URI uri, Parse parse) {
+ public void handle(URI uri, ContentEntity entity) {
visitedLinks.add(uri);
}
@@ -102,7 +101,7 @@
Handler testHandler = new Handler() {
- public void handle(InputStream openStream, URI uri, Parse parse) {
+ public void handle(URI uri, ContentEntity entity) {
visitedLinks.add(uri);
if (uri.getPath().equals("/page3_html")) {
throw new RuntimeException("Oppsie!!!");