You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by to...@apache.org on 2012/12/18 08:48:00 UTC
svn commit: r1423339 [1/5] - in /incubator/droids/branches/0.2.x-cleanup:
droids-crawler/ droids-crawler/src/main/java/org/apache/droids/crawler/
droids-crawler/src/main/java/org/apache/droids/protocol/http/
droids-crawler/src/test/java/org/apache/droi...
Author: tobr
Date: Tue Dec 18 08:47:39 2012
New Revision: 1423339
URL: http://svn.apache.org/viewvc?rev=1423339&view=rev
Log:
reformat code base
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/localserver/LocalHttpServer.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/localserver/ResourceHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
incubator/droids/branches/0.2.x-cleanup/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/AbstractRule.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/AllowedRule.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/ContentLoader.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/DisallowedRule.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/NoRobotException.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/Rule.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/RulesEngine.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/SimpleContentLoader.java
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/package.html
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/site/site.xml
incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/test/java/org/apache/droids/norobots/TestNorobotsClient.java
incubator/droids/branches/0.2.x-cleanup/droids-solr/example/conf/schema.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/example/conf/solrconfig.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/locationmap.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/xdocs/index.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/xdocs/site.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/xdocs/tabs.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/skinconf.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/main/java/org/apache/droids/solr/AdvancedSolrHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/main/java/org/apache/droids/solr/SolrHandler.java
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/site/site.xml
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/test/java/org/apache/droids/solr/AdvancedSolrHandleTest.java
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/test/java/org/apache/droids/solr/MockContentEntity.java
incubator/droids/branches/0.2.x-cleanup/droids-solr/src/test/java/org/apache/droids/solr/SolrHandleTest.java
incubator/droids/branches/0.2.x-cleanup/droids-spring/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/locationmap.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/xdocs/index.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/xdocs/site.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/xdocs/tabs.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/skinconf.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/java/org/apache/droids/dynamic/Cli.java
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/java/org/apache/droids/dynamic/DroidsConfig.java
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-factories-context.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/site/site.xml
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-spring/src/test/resources/droids-core-test-context.xml
incubator/droids/branches/0.2.x-cleanup/droids-tika/pom.xml
incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/TikaDocumentParser.java
incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/TikaHtmlParser.java
incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/api/TikaParse.java
incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/parse/TikaParseImpl.java
incubator/droids/branches/0.2.x-cleanup/droids-tika/src/site/site.xml
incubator/droids/branches/0.2.x-cleanup/droids-tika/src/test/java/org/apache/droids/tika/TikaHtmlParserTest.java
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml Tue Dec 18 08:47:39 2012
@@ -1,25 +1,26 @@
<?xml version="1.0"?>
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.droids</groupId>
- <artifactId>droids</artifactId>
- <version>0.3.0-incubating-SNAPSHOT</version>
- </parent>
- <artifactId>droids-crawler</artifactId>
- <name>droids-crawler</name>
- <dependencies>
- <dependency>
- <groupId>org.apache.droids</groupId>
- <artifactId>droids-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.1</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids</artifactId>
+ <version>0.3.0-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>droids-crawler</artifactId>
+ <name>droids-crawler</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java Tue Dec 18 08:47:39 2012
@@ -21,6 +21,7 @@ import java.net.URISyntaxException;
import java.util.Collection;
import com.google.common.base.Preconditions;
+
import java.util.Queue;
import org.apache.droids.core.AbstractDroid;
@@ -30,42 +31,42 @@ import org.apache.droids.exception.Inval
public abstract class CrawlingDroid extends AbstractDroid<Link> {
- private Collection<String> initialLocations;
+ private Collection<String> initialLocations;
- public CrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
- super(queue, taskMaster);
- }
-
- public void setInitialLocations(Collection<String> initialLocations) {
- this.initialLocations = initialLocations;
- }
-
- @Override
- public void init() throws InvalidTaskException {
- Preconditions.checkState(initialLocations != null,
- "WebCrawlerDroid requires at least one starting file");
- Preconditions.checkState(!initialLocations.isEmpty(),
- "WebCrawlerDroid requires at least one starting file");
- for (String location : initialLocations) {
- URI uri;
- try {
- uri = new URI(location);
- } catch (URISyntaxException ex) {
- throw new InvalidTaskException("Invalid lication: " + location);
- }
- queue.offer(new LinkTask(null, uri, 0));
- }
- }
-
- public void start() {
- taskMaster.start(queue, this);
- }
-
- @Override
- public void finished() {
- logger.info("FINISHED!!!");
- }
+ public CrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ public void setInitialLocations(Collection<String> initialLocations) {
+ this.initialLocations = initialLocations;
+ }
+
+ @Override
+ public void init() throws InvalidTaskException {
+ Preconditions.checkState(initialLocations != null,
+ "WebCrawlerDroid requires at least one starting file");
+ Preconditions.checkState(!initialLocations.isEmpty(),
+ "WebCrawlerDroid requires at least one starting file");
+ for (String location : initialLocations) {
+ URI uri;
+ try {
+ uri = new URI(location);
+ } catch (URISyntaxException ex) {
+ throw new InvalidTaskException("Invalid lication: " + location);
+ }
+ queue.offer(new LinkTask(null, uri, 0));
+ }
+ }
+
+ public void start() {
+ taskMaster.start(queue, this);
+ }
+
+ @Override
+ public void finished() {
+ logger.info("FINISHED!!!");
+ }
- public abstract Worker<Link> getNewWorker();
+ public abstract Worker<Link> getNewWorker();
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java Tue Dec 18 08:47:39 2012
@@ -35,104 +35,104 @@ import org.slf4j.LoggerFactory;
public class CrawlingWorker implements Worker<Link> {
- private static final Logger LOG = LoggerFactory.getLogger(CrawlingWorker.class);
+ private static final Logger LOG = LoggerFactory.getLogger(CrawlingWorker.class);
- private final CrawlingDroid droid;
- HandlerFactory handlerFactory;
+ private final CrawlingDroid droid;
+ HandlerFactory handlerFactory;
- public CrawlingWorker(CrawlingDroid droid) {
- this.droid = droid;
- }
-
- @Override
- public void execute(Link link) throws DroidsException, IOException {
- final String userAgent = this.getClass().getCanonicalName();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting " + userAgent);
- }
- URI uri = link.getURI();
- final Protocol protocol = droid.getProtocolFactory().getProtocol(uri);
- if (protocol == null) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Unsupported protocol scheme '" + uri.getScheme() + "'");
- }
- return;
- }
-
- if (protocol.isAllowed(uri)) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Loading " + uri);
- }
+ public CrawlingWorker(CrawlingDroid droid) {
+ this.droid = droid;
+ }
+
+ @Override
+ public void execute(Link link) throws DroidsException, IOException {
+ final String userAgent = this.getClass().getCanonicalName();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting " + userAgent);
+ }
+ URI uri = link.getURI();
+ final Protocol protocol = droid.getProtocolFactory().getProtocol(uri);
+ if (protocol == null) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Unsupported protocol scheme '" + uri.getScheme() + "'");
+ }
+ return;
+ }
+
+ if (protocol.isAllowed(uri)) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Loading " + uri);
+ }
// ContentEntity entity = null;
- try {
+ try {
// entity = protocol.load(uri);
- } catch (OutOfMemoryError e) {
- LOG.error("Out of memory processing: " + uri + " skipping", e);
- throw new DroidsException(e);
- }
- try {
+ } catch (OutOfMemoryError e) {
+ LOG.error("Out of memory processing: " + uri + " skipping", e);
+ throw new DroidsException(e);
+ }
+ try {
// String contentType = entity.getMimeType();
- String contentType = "";
- if (LOG.isDebugEnabled()) {
- LOG.debug("Content type " + contentType);
- }
- if (contentType == null) {
- LOG.info("Missing content type... can't parse...");
- } else {
- Parser parser = droid.getParserFactory().getParser(contentType);
- if (parser == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not find parser for " + contentType);
- }
- } else {
+ String contentType = "";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Content type " + contentType);
+ }
+ if (contentType == null) {
+ LOG.info("Missing content type... can't parse...");
+ } else {
+ Parser parser = droid.getParserFactory().getParser(contentType);
+ if (parser == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not find parser for " + contentType);
+ }
+ } else {
// Parse parse = parser.parse(entity, link);
- Parse parse = null;
- if (parse.getNewTasks() != null && parse.isFollowed()) {
- Collection<Link> outlinks = getFilteredOutlinks(parse);
- droid.getQueue().addAll(outlinks);
- }
+ Parse parse = null;
+ if (parse.getNewTasks() != null && parse.isFollowed()) {
+ Collection<Link> outlinks = getFilteredOutlinks(parse);
+ droid.getQueue().addAll(outlinks);
+ }
// entity.setParse(parse);
- handle(link);
- }
- }
- } finally {
+ handle(link);
+ }
+ }
+ } finally {
// entity.finish();
- }
- } else {
- if (LOG.isInfoEnabled()) {
- LOG.info("Stopping processing since" + " bots are not allowed for " + uri);
- }
- }
- }
-
- protected void handle(Task task) throws DroidsException, IOException {
- getHandlerFactory().handle(task);
- }
-
- protected Collection<Link> getFilteredOutlinks(Parse parse) {
- URLFiltersFactory filters = droid.getFiltersFactory();
-
- // TODO -- make the hashvalue for Outlink...
- Map<URI, Link> filtered = new LinkedHashMap<URI, Link>();
- for (Task outTask : parse.getNewTasks()) {
- // only use Links, so if for some reason it isn't a Link, skip
- if (!(outTask instanceof Link)) {
- continue;
- }
- Link outlink = (Link) outTask;
- URI uri = outlink.getURI();
- if (filters.accept(outlink) && !filtered.containsKey(uri)) {
- filtered.put(uri, outlink);
- }
- }
- return filtered.values();
- }
-
- public HandlerFactory getHandlerFactory() {
- return handlerFactory;
- }
-
- public void setHandlerFactory(HandlerFactory handlerFactory) {
- this.handlerFactory = handlerFactory;
- }
+ }
+ } else {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Stopping processing since" + " bots are not allowed for " + uri);
+ }
+ }
+ }
+
+ protected void handle(Task task) throws DroidsException, IOException {
+ getHandlerFactory().handle(task);
+ }
+
+ protected Collection<Link> getFilteredOutlinks(Parse parse) {
+ URLFiltersFactory filters = droid.getFiltersFactory();
+
+ // TODO -- make the hashvalue for Outlink...
+ Map<URI, Link> filtered = new LinkedHashMap<URI, Link>();
+ for (Task outTask : parse.getNewTasks()) {
+ // only use Links, so if for some reason it isn't a Link, skip
+ if (!(outTask instanceof Link)) {
+ continue;
+ }
+ Link outlink = (Link) outTask;
+ URI uri = outlink.getURI();
+ if (filters.accept(outlink) && !filtered.containsKey(uri)) {
+ filtered.put(uri, outlink);
+ }
+ }
+ return filtered.values();
+ }
+
+ public HandlerFactory getHandlerFactory() {
+ return handlerFactory;
+ }
+
+ public void setHandlerFactory(HandlerFactory handlerFactory) {
+ this.handlerFactory = handlerFactory;
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java Tue Dec 18 08:47:39 2012
@@ -24,36 +24,35 @@ import org.apache.droids.core.Task;
/**
- * Simple extension of a {@link Task}. Adding from/to link, anchor text
+ * Simple extension of a {@link Task}. Adding from/to link, anchor text
* and the last modified attribute to the task object.
- *
+ *
* @version 1.0
- *
*/
public interface Link extends Task {
- /**
- * @return the Anchor text for this link
- */
- String getAnchorText();
-
- /**
- * From where the link was created
- *
- * @return the parent link from where the link was coming from
- */
- Link getFrom();
+ /**
+ * @return the Anchor text for this link
+ */
+ String getAnchorText();
- /**
- * To where the link is pointing to
- *
- * @return the location where the link is pointing to
- */
- Collection<URI> getTo();
+ /**
+ * From where the link was created
+ *
+ * @return the parent link from where the link was coming from
+ */
+ Link getFrom();
- /**
- * last modified date
- *
- * @return last modified date
- */
- Date getLastModifiedDate();
+ /**
+ * To where the link is pointing to
+ *
+ * @return the location where the link is pointing to
+ */
+ Collection<URI> getTo();
+
+ /**
+ * last modified date
+ *
+ * @return last modified date
+ */
+ Date getLastModifiedDate();
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java Tue Dec 18 08:47:39 2012
@@ -25,171 +25,169 @@ import java.io.Serializable;
/**
- *
* Basic implementation for @Link. LinkTasks are working instructions for URI
* based droids.
- *
*/
public class LinkTask implements Link, Serializable {
- private static final long serialVersionUID = -44808094386453088L;
+ private static final long serialVersionUID = -44808094386453088L;
- private Date started;
- private final int depth;
- private final URI uri;
- private final Link from;
-
- private Date lastModifiedDate;
- private Collection<URI> linksTo;
- private String anchorText;
- private int weight;
- private boolean aborted = false;
-
- /**
- * Creates a new LinkTask.
- *
- * @param from
- * @param uri
- * @param depth
- */
- public LinkTask(Link from, URI uri, int depth) {
- this.from = from;
- this.uri = uri;
- this.depth = depth;
- this.started = new Date();
- }
-
- /**
- * Creates a new LinkTask.
- *
- * @param from
- * @param uri
- * @param depth
- * @param weight
- */
- public LinkTask(Link from, URI uri, int depth, int weight) {
- this.from = from;
- this.uri = uri;
- this.depth = depth;
- this.started = new Date();
- this.weight = weight;
- }
-
- /**
- * Creates a new LinkTask
- *
- * @param from
- * @param uri
- * @param depth
- * @param anchorText
- */
- public LinkTask(Link from, URI uri, int depth, String anchorText) {
- this(from, uri, depth);
- this.anchorText = anchorText;
- }
-
- @Override
- public URI getURI() {
- return uri;
- }
-
- @Override
- public InputStream getContent() throws IOException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Date getTaskDate() {
- return started;
- }
-
- /**
- * Set the Date the task started.
- *
- * @param started
- */
- public void setTaskDate(Date started) {
- this.started = started;
- }
-
- @Override
- public int getDepth() {
- return depth;
- }
-
- @Override
- public Link getFrom() {
- return from;
- }
-
- @Override
- public Collection<URI> getTo() {
- return linksTo;
- }
-
- @Override
- public Date getLastModifiedDate() {
- return lastModifiedDate;
- }
-
- /**
- * Set the Date the Task object was last modified.
- *
- * @param lastModifiedDate
- */
- public void setLastModifiedDate(Date lastModifiedDate) {
- this.lastModifiedDate = lastModifiedDate;
- }
-
- /**
- * Set Outgoing links.
- *
- * @param linksTo
- */
- public void setLinksTo(Collection<URI> linksTo) {
- this.linksTo = linksTo;
- }
-
- @Override
- public String getAnchorText() {
- return anchorText;
- }
-
- /**
- * Set the anchor text for this link.
- *
- * @param anchorText
- */
- public void setAnchorText(String anchorText) {
- this.anchorText = anchorText;
- }
-
- /**
- * Get the weight of the link
- *
- * @return the links weight
- */
- public int getWeight() {
- return weight;
- }
-
- /**
- * Set the weight of the link.
- *
- * @param weight
- */
- public void setWeight(int weight) {
- this.weight = weight;
- }
-
- @Override
- public void abort() {
- aborted = true;
- }
-
- @Override
- public boolean isAborted() {
- return aborted;
- }
+ private Date started;
+ private final int depth;
+ private final URI uri;
+ private final Link from;
+
+ private Date lastModifiedDate;
+ private Collection<URI> linksTo;
+ private String anchorText;
+ private int weight;
+ private boolean aborted = false;
+
+ /**
+ * Creates a new LinkTask.
+ *
+ * @param from
+ * @param uri
+ * @param depth
+ */
+ public LinkTask(Link from, URI uri, int depth) {
+ this.from = from;
+ this.uri = uri;
+ this.depth = depth;
+ this.started = new Date();
+ }
+
+ /**
+ * Creates a new LinkTask.
+ *
+ * @param from
+ * @param uri
+ * @param depth
+ * @param weight
+ */
+ public LinkTask(Link from, URI uri, int depth, int weight) {
+ this.from = from;
+ this.uri = uri;
+ this.depth = depth;
+ this.started = new Date();
+ this.weight = weight;
+ }
+
+ /**
+ * Creates a new LinkTask
+ *
+ * @param from
+ * @param uri
+ * @param depth
+ * @param anchorText
+ */
+ public LinkTask(Link from, URI uri, int depth, String anchorText) {
+ this(from, uri, depth);
+ this.anchorText = anchorText;
+ }
+
+ @Override
+ public URI getURI() {
+ return uri;
+ }
+
+ @Override
+ public InputStream getContent() throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Date getTaskDate() {
+ return started;
+ }
+
+ /**
+ * Set the Date the task started.
+ *
+ * @param started
+ */
+ public void setTaskDate(Date started) {
+ this.started = started;
+ }
+
+ @Override
+ public int getDepth() {
+ return depth;
+ }
+
+ @Override
+ public Link getFrom() {
+ return from;
+ }
+
+ @Override
+ public Collection<URI> getTo() {
+ return linksTo;
+ }
+
+ @Override
+ public Date getLastModifiedDate() {
+ return lastModifiedDate;
+ }
+
+ /**
+ * Set the Date the Task object was last modified.
+ *
+ * @param lastModifiedDate
+ */
+ public void setLastModifiedDate(Date lastModifiedDate) {
+ this.lastModifiedDate = lastModifiedDate;
+ }
+
+ /**
+ * Set Outgoing links.
+ *
+ * @param linksTo
+ */
+ public void setLinksTo(Collection<URI> linksTo) {
+ this.linksTo = linksTo;
+ }
+
+ @Override
+ public String getAnchorText() {
+ return anchorText;
+ }
+
+ /**
+ * Set the anchor text for this link.
+ *
+ * @param anchorText
+ */
+ public void setAnchorText(String anchorText) {
+ this.anchorText = anchorText;
+ }
+
+ /**
+ * Get the weight of the link
+ *
+ * @return the links weight
+ */
+ public int getWeight() {
+ return weight;
+ }
+
+ /**
+ * Set the weight of the link.
+ *
+ * @param weight
+ */
+ public void setWeight(int weight) {
+ this.weight = weight;
+ }
+
+ @Override
+ public void abort() {
+ aborted = true;
+ }
+
+ @Override
+ public boolean isAborted() {
+ return aborted;
+ }
}
\ No newline at end of file
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java Tue Dec 18 08:47:39 2012
@@ -19,6 +19,7 @@
package org.apache.droids.crawler;
import java.util.Queue;
+
import org.apache.droids.core.TaskMaster;
import org.apache.droids.core.Worker;
import org.apache.droids.handle.ReportHandler;
@@ -26,23 +27,21 @@ import org.apache.droids.helper.factorie
/**
- *
* This simple CrawlingDroid uses the ReportHandler to handle all retrieved files.
- *
*/
public class ReportCrawlingDroid extends CrawlingDroid {
- public ReportCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
- super(queue, taskMaster);
- }
-
- @Override
- public Worker<Link> getNewWorker() {
- final CrawlingWorker worker = new CrawlingWorker( this );
- HandlerFactory handlerFactory = new HandlerFactory();
- handlerFactory.getMap().put("default", new ReportHandler());
- worker.setHandlerFactory(handlerFactory);
- return worker;
- }
+ public ReportCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker(this);
+ HandlerFactory handlerFactory = new HandlerFactory();
+ handlerFactory.getMap().put("default", new ReportHandler());
+ worker.setHandlerFactory(handlerFactory);
+ return worker;
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java Tue Dec 18 08:47:39 2012
@@ -18,51 +18,51 @@
*/
package org.apache.droids.protocol.http;
+import org.apache.droids.core.AdvancedManagedContentEntity;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import org.apache.droids.core.AdvancedManagedContentEntity;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-
public class AdvancedHttpContentEntity extends HttpContentEntity implements
- AdvancedManagedContentEntity {
+ AdvancedManagedContentEntity {
+
+ private Map<String, String> metadata = new HashMap<String, String>();
+ private long contentLength;
+
+ public AdvancedHttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
+ super(entity, maxlen);
+ }
+
+ public AdvancedHttpContentEntity(HttpEntity entity, Header[] allHeaders, long maxlen) throws IOException {
+ super(entity, maxlen);
+ for (Header h : allHeaders) {
+ metadata.put(h.getName(), h.getValue());
+ }
+ contentLength = entity.getContentLength();
+ }
+
+ @Override
+ public boolean containsMetadataKey(String key) {
+ return metadata.containsKey(key);
+ }
- private Map<String,String> metadata = new HashMap<String,String>();
- private long contentLength;
-
- public AdvancedHttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
- super(entity, maxlen);
- }
-
- public AdvancedHttpContentEntity(HttpEntity entity, Header[] allHeaders, long maxlen) throws IOException {
- super(entity, maxlen);
- for(Header h : allHeaders) {
- metadata.put(h.getName(), h.getValue());
- }
- contentLength = entity.getContentLength();
- }
-
- @Override
- public boolean containsMetadataKey(String key) {
- return metadata.containsKey(key);
- }
-
- @Override
- public String getValue(String key) {
- return metadata.get(key);
- }
-
- @Override
- public Set<String> metadataKeySet() {
- return metadata.keySet();
- }
-
- @Override
- public long getContentLength() {
- return this.contentLength;
- }
+ @Override
+ public String getValue(String key) {
+ return metadata.get(key);
+ }
+
+ @Override
+ public Set<String> metadataKeySet() {
+ return metadata.keySet();
+ }
+
+ @Override
+ public long getContentLength() {
+ return this.contentLength;
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java Tue Dec 18 08:47:39 2012
@@ -33,32 +33,32 @@ import org.apache.http.client.methods.Ht
public class AdvancedHttpProtocol extends HttpProtocol {
- public AdvancedHttpProtocol(HttpClient httpclient) {
- super(httpclient);
- }
-
- public AdvancedHttpProtocol() {
- super();
- }
-
- @Override
- public AdvancedManagedContentEntity load(URI uri) throws IOException {
- HttpGet httpget = new HttpGet(uri);
- HttpResponse response = getHttpClient().execute(httpget);
- StatusLine statusline = response.getStatusLine();
- if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
- httpget.abort();
- throw new HttpResponseException(
- statusline.getStatusCode(), statusline.getReasonPhrase());
+ public AdvancedHttpProtocol(HttpClient httpclient) {
+ super(httpclient);
+ }
+
+ public AdvancedHttpProtocol() {
+ super();
}
- HttpEntity entity = response.getEntity();
- if (entity == null) {
- // Should _almost_ never happen with HTTP GET requests.
- throw new ClientProtocolException("Empty entity");
+
+ @Override
+ public AdvancedManagedContentEntity load(URI uri) throws IOException {
+ HttpGet httpget = new HttpGet(uri);
+ HttpResponse response = getHttpClient().execute(httpget);
+ StatusLine statusline = response.getStatusLine();
+ if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
+ httpget.abort();
+ throw new HttpResponseException(
+ statusline.getStatusCode(), statusline.getReasonPhrase());
+ }
+ HttpEntity entity = response.getEntity();
+ if (entity == null) {
+ // Should _almost_ never happen with HTTP GET requests.
+ throw new ClientProtocolException("Empty entity");
+ }
+ long maxlen = getHttpClient().getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
+ return new AdvancedHttpContentEntity(entity, response.getAllHeaders(), maxlen);
}
- long maxlen = getHttpClient().getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
- return new AdvancedHttpContentEntity(entity,response.getAllHeaders(),maxlen);
- }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java Tue Dec 18 08:47:39 2012
@@ -18,14 +18,12 @@ package org.apache.droids.protocol.http;
import java.io.IOException;
-public class ContentTooLongException extends IOException
-{
+public class ContentTooLongException extends IOException {
- private static final long serialVersionUID = -3118026295438863279L;
+ private static final long serialVersionUID = -3118026295438863279L;
+
+ public ContentTooLongException(String message) {
+ super(message);
+ }
- public ContentTooLongException(String message)
- {
- super(message);
- }
-
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java Tue Dec 18 08:47:39 2012
@@ -27,88 +27,69 @@ import org.apache.http.HttpEntity;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.util.ByteArrayBuffer;
-class DroidHttpEntity extends HttpEntityWrapper
-{
- private final byte[] buffer;
-
- public DroidHttpEntity(final HttpEntity entity, long maxlen) throws IOException
- {
- super(entity);
- if (!entity.isRepeatable() || entity.getContentLength() < 0)
- {
- InputStream instream = entity.getContent();
- ByteArrayBuffer buf = new ByteArrayBuffer(4096);
- try
- {
- byte[] tmp = new byte[4096];
- long total = 0;
- int l;
- while ((l = instream.read(tmp)) != -1)
- {
- buf.append(tmp, 0, l);
- total += l;
- if (maxlen > 0 && total >= maxlen) {
- throw new ContentTooLongException("Content length exceeds " + maxlen + " byte limit");
- }
+class DroidHttpEntity extends HttpEntityWrapper {
+ private final byte[] buffer;
+
+ public DroidHttpEntity(final HttpEntity entity, long maxlen) throws IOException {
+ super(entity);
+ if (!entity.isRepeatable() || entity.getContentLength() < 0) {
+ InputStream instream = entity.getContent();
+ ByteArrayBuffer buf = new ByteArrayBuffer(4096);
+ try {
+ byte[] tmp = new byte[4096];
+ long total = 0;
+ int l;
+ while ((l = instream.read(tmp)) != -1) {
+ buf.append(tmp, 0, l);
+ total += l;
+ if (maxlen > 0 && total >= maxlen) {
+ throw new ContentTooLongException("Content length exceeds " + maxlen + " byte limit");
+ }
+ }
+ this.buffer = buf.toByteArray();
+ } finally {
+ instream.close();
+ }
+ } else {
+ this.buffer = null;
}
- this.buffer = buf.toByteArray();
- } finally
- {
- instream.close();
- }
- } else
- {
- this.buffer = null;
- }
- }
-
- public long getContentLength()
- {
- if (this.buffer != null)
- {
- return this.buffer.length;
- } else
- {
- return wrappedEntity.getContentLength();
- }
- }
-
- public InputStream getContent() throws IOException
- {
- if (this.buffer != null)
- {
- return new ByteArrayInputStream(this.buffer);
- } else
- {
- return wrappedEntity.getContent();
- }
- }
-
- public boolean isChunked()
- {
- return (buffer == null) && wrappedEntity.isChunked();
- }
-
- public boolean isRepeatable()
- {
- return true;
- }
-
- public void writeTo(final OutputStream outstream) throws IOException
- {
- Preconditions.checkArgument( outstream != null, "Output stream may not be null" );
- if (this.buffer != null)
- {
- outstream.write(this.buffer);
- } else
- {
- wrappedEntity.writeTo(outstream);
- }
- }
-
- public boolean isStreaming()
- {
- return (buffer == null) && wrappedEntity.isStreaming();
- }
+ }
+
+ public long getContentLength() {
+ if (this.buffer != null) {
+ return this.buffer.length;
+ } else {
+ return wrappedEntity.getContentLength();
+ }
+ }
+
+ public InputStream getContent() throws IOException {
+ if (this.buffer != null) {
+ return new ByteArrayInputStream(this.buffer);
+ } else {
+ return wrappedEntity.getContent();
+ }
+ }
+
+ public boolean isChunked() {
+ return (buffer == null) && wrappedEntity.isChunked();
+ }
+
+ public boolean isRepeatable() {
+ return true;
+ }
+
+ public void writeTo(final OutputStream outstream) throws IOException {
+ Preconditions.checkArgument(outstream != null, "Output stream may not be null");
+ if (this.buffer != null) {
+ outstream.write(this.buffer);
+ } else {
+ wrappedEntity.writeTo(outstream);
+ }
+ }
+
+ public boolean isStreaming() {
+ return (buffer == null) && wrappedEntity.isStreaming();
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java Tue Dec 18 08:47:39 2012
@@ -70,122 +70,110 @@ import org.apache.http.protocol.RequestU
/**
* {@link HttpClient} implementation optimized specifically for web crawling.
* This HTTP agent has no support for HTTP state management and authentication
- * and is expected to be used for retrieving information from publicly
- * accessible sites using stateless, idempotent HTTP methods.
+ * and is expected to be used for retrieving information from publicly
+ * accessible sites using stateless, idempotent HTTP methods.
*/
-public class DroidsHttpClient extends AbstractHttpClient
-{
+public class DroidsHttpClient extends AbstractHttpClient {
- public static final String MAX_BODY_LENGTH = "droids.http..max-body-length";
-
- public DroidsHttpClient()
- {
- super(null, null);
- }
-
- public DroidsHttpClient(HttpParams params)
- {
- super(null, params);
- }
-
- @Override
- protected HttpParams createHttpParams()
- {
- HttpParams params = new BasicHttpParams();
- params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
- params.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, HTTP.DEFAULT_CONTENT_CHARSET);
- params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
- params.setParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
- params.setIntParameter(CoreConnectionPNames.MAX_HEADER_COUNT, 256);
- params.setIntParameter(CoreConnectionPNames.MAX_LINE_LENGTH, 5 * 1024);
- params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 20000);
- params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000);
- params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
- //params.setLongParameter(MAX_BODY_LENGTH, 512 * 1024);
- return params;
- }
-
- @Override
- protected BasicHttpProcessor createHttpProcessor()
- {
- BasicHttpProcessor httpproc = new BasicHttpProcessor();
- httpproc.addInterceptor(new RequestDefaultHeaders());
- // Required protocol interceptors
- httpproc.addInterceptor(new RequestContent());
- httpproc.addInterceptor(new RequestTargetHost());
- // Recommended protocol interceptors
- httpproc.addInterceptor(new RequestConnControl());
- httpproc.addInterceptor(new RequestUserAgent());
- httpproc.addInterceptor(new RequestExpectContinue());
- // HTTP authentication interceptors
- httpproc.addInterceptor(new RequestProxyAuthentication());
- return httpproc;
- }
-
- @Override
- protected ClientConnectionManager createClientConnectionManager()
- {
- SchemeRegistry schemeRegistry = new SchemeRegistry();
- schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
- schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
- return new DroidsHttpConnectionManager(getParams(), schemeRegistry);
- }
-
- @Override
- protected AuthSchemeRegistry createAuthSchemeRegistry()
- {
- AuthSchemeRegistry registry = new AuthSchemeRegistry();
- registry.register(
- AuthPolicy.BASIC,
- new BasicSchemeFactory());
- registry.register(
- AuthPolicy.DIGEST,
- new DigestSchemeFactory());
- return registry;
- }
-
- @Override
- protected CookieSpecRegistry createCookieSpecRegistry()
- {
- // Return empty cookie scheme registry. There'll be no cookie support
- return new CookieSpecRegistry();
- }
-
-
- @Override
- protected HttpContext createHttpContext()
- {
- HttpContext context = new BasicHttpContext();
- context.setAttribute(
- ClientContext.AUTHSCHEME_REGISTRY,
- getAuthSchemes());
- context.setAttribute(
- ClientContext.CREDS_PROVIDER,
- getCredentialsProvider());
- return context;
- }
-
- @Override
- protected HttpRequestRetryHandler createHttpRequestRetryHandler()
- {
- return new DroidsRequestRetryHandler();
- }
-
- /**
- * Added ProxySelectorRoutePlanner to support JVM Proxy Settings
- */
- @Override
- protected HttpRoutePlanner createHttpRoutePlanner()
- {
- return new ProxySelectorRoutePlanner(
- this.getConnectionManager().getSchemeRegistry(),
- ProxySelector.getDefault());
- }
-
- @Override
- protected AuthenticationHandler createTargetAuthenticationHandler()
- {
- return new NoAuthHandler();
- }
+ public static final String MAX_BODY_LENGTH = "droids.http..max-body-length";
+
+ public DroidsHttpClient() {
+ super(null, null);
+ }
+
+ public DroidsHttpClient(HttpParams params) {
+ super(null, params);
+ }
+
+ @Override
+ protected HttpParams createHttpParams() {
+ HttpParams params = new BasicHttpParams();
+ params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
+ params.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, HTTP.DEFAULT_CONTENT_CHARSET);
+ params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
+ params.setParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
+ params.setIntParameter(CoreConnectionPNames.MAX_HEADER_COUNT, 256);
+ params.setIntParameter(CoreConnectionPNames.MAX_LINE_LENGTH, 5 * 1024);
+ params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 20000);
+ params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000);
+ params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
+ //params.setLongParameter(MAX_BODY_LENGTH, 512 * 1024);
+ return params;
+ }
+
+ @Override
+ protected BasicHttpProcessor createHttpProcessor() {
+ BasicHttpProcessor httpproc = new BasicHttpProcessor();
+ httpproc.addInterceptor(new RequestDefaultHeaders());
+ // Required protocol interceptors
+ httpproc.addInterceptor(new RequestContent());
+ httpproc.addInterceptor(new RequestTargetHost());
+ // Recommended protocol interceptors
+ httpproc.addInterceptor(new RequestConnControl());
+ httpproc.addInterceptor(new RequestUserAgent());
+ httpproc.addInterceptor(new RequestExpectContinue());
+ // HTTP authentication interceptors
+ httpproc.addInterceptor(new RequestProxyAuthentication());
+ return httpproc;
+ }
+
+ @Override
+ protected ClientConnectionManager createClientConnectionManager() {
+ SchemeRegistry schemeRegistry = new SchemeRegistry();
+ schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
+ schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
+ return new DroidsHttpConnectionManager(getParams(), schemeRegistry);
+ }
+
+ @Override
+ protected AuthSchemeRegistry createAuthSchemeRegistry() {
+ AuthSchemeRegistry registry = new AuthSchemeRegistry();
+ registry.register(
+ AuthPolicy.BASIC,
+ new BasicSchemeFactory());
+ registry.register(
+ AuthPolicy.DIGEST,
+ new DigestSchemeFactory());
+ return registry;
+ }
+
+ @Override
+ protected CookieSpecRegistry createCookieSpecRegistry() {
+ // Return empty cookie scheme registry. There'll be no cookie support
+ return new CookieSpecRegistry();
+ }
+
+
+ @Override
+ protected HttpContext createHttpContext() {
+ HttpContext context = new BasicHttpContext();
+ context.setAttribute(
+ ClientContext.AUTHSCHEME_REGISTRY,
+ getAuthSchemes());
+ context.setAttribute(
+ ClientContext.CREDS_PROVIDER,
+ getCredentialsProvider());
+ return context;
+ }
+
+ @Override
+ protected HttpRequestRetryHandler createHttpRequestRetryHandler() {
+ return new DroidsRequestRetryHandler();
+ }
+
+ /**
+ * Added ProxySelectorRoutePlanner to support JVM Proxy Settings
+ */
+ @Override
+ protected HttpRoutePlanner createHttpRoutePlanner() {
+ return new ProxySelectorRoutePlanner(
+ this.getConnectionManager().getSchemeRegistry(),
+ ProxySelector.getDefault());
+ }
+
+ @Override
+ protected AuthenticationHandler createTargetAuthenticationHandler() {
+ return new NoAuthHandler();
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java Tue Dec 18 08:47:39 2012
@@ -23,25 +23,22 @@ import org.apache.http.io.HttpMessagePar
import org.apache.http.io.SessionInputBuffer;
import org.apache.http.params.HttpParams;
-class DroidsHttpConnection extends DefaultClientConnection
-{
+class DroidsHttpConnection extends DefaultClientConnection {
- public DroidsHttpConnection()
- {
- super();
- }
+ public DroidsHttpConnection() {
+ super();
+ }
- @Override
- protected HttpMessageParser createResponseParser(
- SessionInputBuffer buffer,
- HttpResponseFactory responseFactory,
- HttpParams params)
- {
- return new DefaultResponseParser(
- buffer,
- new LenientHttpResponseParser(),
- responseFactory,
- params);
- }
+ @Override
+ protected HttpMessageParser createResponseParser(
+ SessionInputBuffer buffer,
+ HttpResponseFactory responseFactory,
+ HttpParams params) {
+ return new DefaultResponseParser(
+ buffer,
+ new LenientHttpResponseParser(),
+ responseFactory,
+ params);
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java Tue Dec 18 08:47:39 2012
@@ -21,18 +21,15 @@ import org.apache.http.conn.scheme.Schem
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.HttpParams;
-class DroidsHttpConnectionManager extends ThreadSafeClientConnManager
-{
+class DroidsHttpConnectionManager extends ThreadSafeClientConnManager {
- public DroidsHttpConnectionManager(HttpParams params, SchemeRegistry schemes)
- {
- super(params, schemes);
- }
+ public DroidsHttpConnectionManager(HttpParams params, SchemeRegistry schemes) {
+ super(params, schemes);
+ }
+
+ @Override
+ protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schemes) {
+ return new DroidsHttpConnectionOperator(schemes);
+ }
- @Override
- protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schemes)
- {
- return new DroidsHttpConnectionOperator(schemes);
- }
-
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java Tue Dec 18 08:47:39 2012
@@ -20,18 +20,15 @@ import org.apache.http.conn.OperatedClie
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.conn.DefaultClientConnectionOperator;
-class DroidsHttpConnectionOperator extends DefaultClientConnectionOperator
-{
+class DroidsHttpConnectionOperator extends DefaultClientConnectionOperator {
- public DroidsHttpConnectionOperator(SchemeRegistry schemes)
- {
- super(schemes);
- }
+ public DroidsHttpConnectionOperator(SchemeRegistry schemes) {
+ super(schemes);
+ }
- @Override
- public OperatedClientConnection createConnection()
- {
- return new DroidsHttpConnection();
- }
+ @Override
+ public OperatedClientConnection createConnection() {
+ return new DroidsHttpConnection();
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java Tue Dec 18 08:47:39 2012
@@ -18,7 +18,7 @@ package org.apache.droids.protocol.http;
import java.io.IOException;
import java.io.InterruptedIOException;
- import java.net.UnknownHostException;
+import java.net.UnknownHostException;
import javax.net.ssl.SSLHandshakeException;
@@ -29,55 +29,51 @@ import org.apache.http.client.HttpReques
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.protocol.HttpContext;
-class DroidsRequestRetryHandler implements HttpRequestRetryHandler
-{
+class DroidsRequestRetryHandler implements HttpRequestRetryHandler {
- private final int retryCount;
+ private final int retryCount;
- public DroidsRequestRetryHandler(int retryCount)
- {
- super();
- this.retryCount = retryCount;
- }
-
- public DroidsRequestRetryHandler()
- {
- this(3);
- }
-
- @Override
- public boolean retryRequest(final IOException exception, int executionCount,
- final HttpContext context)
- {
- Preconditions.checkArgument(exception != null, "Exception parameter may not be null" );
- Preconditions.checkArgument(context != null, "HTTP context may not be null" );
-
- if (executionCount > this.retryCount) {
- // Do not retry if over max retry count
- return false;
+ public DroidsRequestRetryHandler(int retryCount) {
+ super();
+ this.retryCount = retryCount;
+ }
+
+ public DroidsRequestRetryHandler() {
+ this(3);
+ }
+
+ @Override
+ public boolean retryRequest(final IOException exception, int executionCount,
+ final HttpContext context) {
+ Preconditions.checkArgument(exception != null, "Exception parameter may not be null");
+ Preconditions.checkArgument(context != null, "HTTP context may not be null");
+
+ if (executionCount > this.retryCount) {
+ // Do not retry if over max retry count
+ return false;
+ }
+ if (exception instanceof NoHttpResponseException) {
+ // Retry if the server dropped connection on us
+ return true;
+ }
+ if (exception instanceof InterruptedIOException) {
+ // Timeout
+ return false;
+ }
+ if (exception instanceof UnknownHostException) {
+ // Unknown host
+ return false;
+ }
+ if (exception instanceof HttpHostConnectException) {
+ // Connection refused
+ return false;
+ }
+ if (exception instanceof SSLHandshakeException) {
+ // SSL handshake exception
+ return false;
+ }
+ // otherwise retry
+ return true;
}
- if (exception instanceof NoHttpResponseException) {
- // Retry if the server dropped connection on us
- return true;
- }
- if (exception instanceof InterruptedIOException) {
- // Timeout
- return false;
- }
- if (exception instanceof UnknownHostException) {
- // Unknown host
- return false;
- }
- if (exception instanceof HttpHostConnectException) {
- // Connection refused
- return false;
- }
- if (exception instanceof SSLHandshakeException) {
- // SSL handshake exception
- return false;
- }
- // otherwise retry
- return true;
- }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java Tue Dec 18 08:47:39 2012
@@ -36,40 +36,38 @@ import org.apache.http.client.methods.Ht
/**
* {@link ContentLoader} based on HttpClient 4.0.
*/
-public class HttpClientContentLoader implements ContentLoader
-{
+public class HttpClientContentLoader implements ContentLoader {
- private final HttpClient httpclient;
-
- public HttpClientContentLoader(HttpClient httpclient) {
- super();
- this.httpclient = httpclient;
- }
-
- public boolean exists(URI uri) throws IOException
- {
- HttpHead httphead = new HttpHead(uri);
- HttpResponse response = httpclient.execute(httphead);
- return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
- }
+ private final HttpClient httpclient;
- public InputStream load(URI uri) throws IOException {
- HttpGet httpget = new HttpGet(uri);
- HttpResponse response = httpclient.execute(httpget);
- StatusLine statusline = response.getStatusLine();
- if (statusline.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
- return null;
+ public HttpClientContentLoader(HttpClient httpclient) {
+ super();
+ this.httpclient = httpclient;
}
- if (statusline.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpResponseException(
- statusline.getStatusCode(), statusline.getReasonPhrase());
+
+ public boolean exists(URI uri) throws IOException {
+ HttpHead httphead = new HttpHead(uri);
+ HttpResponse response = httpclient.execute(httphead);
+ return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
}
- HttpEntity entity = response.getEntity();
- if (entity != null) {
- return entity.getContent();
- } else {
- return null;
+
+ public InputStream load(URI uri) throws IOException {
+ HttpGet httpget = new HttpGet(uri);
+ HttpResponse response = httpclient.execute(httpget);
+ StatusLine statusline = response.getStatusLine();
+ if (statusline.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
+ return null;
+ }
+ if (statusline.getStatusCode() != HttpStatus.SC_OK) {
+ throw new HttpResponseException(
+ statusline.getStatusCode(), statusline.getReasonPhrase());
+ }
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ return entity.getContent();
+ } else {
+ return null;
+ }
}
- }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java Tue Dec 18 08:47:39 2012
@@ -30,77 +30,77 @@ import org.apache.http.protocol.HTTP;
public class HttpContentEntity implements ManagedContentEntity {
- private final HttpEntity entity;
- private final String mimeType;
- private final String charset;
-
- private Parse parse = null;
-
- public HttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
- super();
- if (entity.isRepeatable()) {
- this.entity = entity;
- } else {
- this.entity = new DroidHttpEntity(entity, maxlen);
- }
-
- String mimeType = null;
- String charset = null;
- Header header = entity.getContentType();
- if (header != null) {
- HeaderElement[] helems = header.getElements();
- if (helems != null && helems.length > 0) {
- HeaderElement helem = helems[0];
- mimeType = helem.getName();
- NameValuePair nvp = helem.getParameterByName("charset");
- if (nvp != null) {
- charset = nvp.getValue();
+ private final HttpEntity entity;
+ private final String mimeType;
+ private final String charset;
+
+ private Parse parse = null;
+
+ public HttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
+ super();
+ if (entity.isRepeatable()) {
+ this.entity = entity;
+ } else {
+ this.entity = new DroidHttpEntity(entity, maxlen);
}
- }
+
+ String mimeType = null;
+ String charset = null;
+ Header header = entity.getContentType();
+ if (header != null) {
+ HeaderElement[] helems = header.getElements();
+ if (helems != null && helems.length > 0) {
+ HeaderElement helem = helems[0];
+ mimeType = helem.getName();
+ NameValuePair nvp = helem.getParameterByName("charset");
+ if (nvp != null) {
+ charset = nvp.getValue();
+ }
+ }
+ }
+ if (mimeType != null) {
+ this.mimeType = mimeType.toLowerCase(Locale.ENGLISH);
+ } else {
+ this.mimeType = "binary/octet-stream";
+ }
+ if (charset != null) {
+ this.charset = charset;
+ } else {
+ if (this.mimeType.startsWith("text/")) {
+ this.charset = HTTP.ISO_8859_1;
+ } else {
+ this.charset = null;
+ }
+ }
+ }
+
+ @Override
+ public String getMimeType() {
+ return mimeType;
+ }
+
+ @Override
+ public String getCharset() {
+ return charset;
+ }
+
+ @Override
+ public InputStream obtainContent() throws IOException {
+ return entity.getContent();
+ }
+
+ @Override
+ public Parse getParse() {
+ return this.parse;
+ }
+
+ @Override
+ public void setParse(Parse parse) {
+ this.parse = parse;
+ }
+
+ @Override
+ public void finish() {
}
- if (mimeType != null) {
- this.mimeType = mimeType.toLowerCase(Locale.ENGLISH);
- } else {
- this.mimeType = "binary/octet-stream";
- }
- if (charset != null) {
- this.charset = charset;
- } else {
- if (this.mimeType.startsWith("text/")) {
- this.charset = HTTP.ISO_8859_1;
- } else {
- this.charset = null;
- }
- }
- }
-
- @Override
- public String getMimeType() {
- return mimeType;
- }
-
- @Override
- public String getCharset() {
- return charset;
- }
-
- @Override
- public InputStream obtainContent() throws IOException {
- return entity.getContent();
- }
-
- @Override
- public Parse getParse() {
- return this.parse;
- }
-
- @Override
- public void setParse(Parse parse) {
- this.parse = parse;
- }
-
- @Override
- public void finish() {
- }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java Tue Dec 18 08:47:39 2012
@@ -42,111 +42,110 @@ import org.slf4j.LoggerFactory;
*/
public class HttpProtocol implements Protocol {
- private static final Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
- private final HttpClient httpclient;
- private final ContentLoader contentLoader;
-
- private boolean forceAllow = false;
- private String userAgent = "Apache-Droids/1.1 (java 1.5)";
-
- public HttpProtocol(final HttpClient httpclient) {
- super();
- this.httpclient = httpclient;
- this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
- this.contentLoader = new HttpClientContentLoader(httpclient);
- }
-
- public HttpProtocol() {
- this(new DroidsHttpClient());
- }
-
- @Override
- public ManagedContentEntity load(URI uri) throws IOException {
- HttpGet httpget = new HttpGet(uri);
- HttpResponse response = httpclient.execute(httpget);
- StatusLine statusline = response.getStatusLine();
- if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
- httpget.abort();
- throw new HttpResponseException(
- statusline.getStatusCode(), statusline.getReasonPhrase());
- }
- HttpEntity entity = response.getEntity();
- if (entity == null) {
- // Should _almost_ never happen with HTTP GET requests.
- throw new ClientProtocolException("Empty entity");
- }
- long maxlen = httpclient.getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
- return new HttpContentEntity(entity, maxlen);
- }
-
- @Override
- public boolean isAllowed(URI uri) throws IOException {
- if (forceAllow) {
- return forceAllow;
- }
-
- URI baseURI;
- try {
- baseURI = new URI(
- uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(),
- "/", null, null);
- } catch (URISyntaxException ex) {
- LOG.error("Unable to determine base URI for " + uri);
- return false;
- }
-
- NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
- try {
- nrc.parse(baseURI);
- } catch (NoRobotException ex) {
- LOG.error("Failure parsing robots.txt: " + ex.getMessage());
- return false;
- }
- boolean test = nrc.isUrlAllowed(uri);
- if (LOG.isInfoEnabled()) {
- LOG.info(uri + " is " + (test ? "allowed" : "denied"));
- }
- return test;
- }
-
- public String getUserAgent() {
- return userAgent;
- }
-
- public void setUserAgent(String userAgent) {
- this.userAgent = userAgent;
- this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
- }
-
- /**
- * You can force that a site is allowed (ignoring the robots.txt). This should
- * only be used on server that you control and where you have the permission
- * to ignore the robots.txt.
- *
- * @return <code>true</code> if you are rude and ignore robots.txt.
- * <code>false</code> if you are playing nice.
- */
- public boolean isForceAllow() {
- return forceAllow;
- }
-
- /**
- * You can force that a site is allowed (ignoring the robot.txt). This should
- * only be used on server that you control and where you have the permission
- * to ignore the robots.txt.
- *
- * @param forceAllow
- * if you want to force an allow and ignore the robot.txt set
- * to <code>true</code>. If you want to obey the rules and
- * be polite set to <code>false</code>.
- */
- public void setForceAllow(boolean forceAllow) {
- this.forceAllow = forceAllow;
- }
-
- protected HttpClient getHttpClient() {
- return this.httpclient;
- }
+ private final HttpClient httpclient;
+ private final ContentLoader contentLoader;
+
+ private boolean forceAllow = false;
+ private String userAgent = "Apache-Droids/1.1 (java 1.5)";
+
+ public HttpProtocol(final HttpClient httpclient) {
+ super();
+ this.httpclient = httpclient;
+ this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
+ this.contentLoader = new HttpClientContentLoader(httpclient);
+ }
+
+ public HttpProtocol() {
+ this(new DroidsHttpClient());
+ }
+
+ @Override
+ public ManagedContentEntity load(URI uri) throws IOException {
+ HttpGet httpget = new HttpGet(uri);
+ HttpResponse response = httpclient.execute(httpget);
+ StatusLine statusline = response.getStatusLine();
+ if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
+ httpget.abort();
+ throw new HttpResponseException(
+ statusline.getStatusCode(), statusline.getReasonPhrase());
+ }
+ HttpEntity entity = response.getEntity();
+ if (entity == null) {
+ // Should _almost_ never happen with HTTP GET requests.
+ throw new ClientProtocolException("Empty entity");
+ }
+ long maxlen = httpclient.getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
+ return new HttpContentEntity(entity, maxlen);
+ }
+
+ @Override
+ public boolean isAllowed(URI uri) throws IOException {
+ if (forceAllow) {
+ return forceAllow;
+ }
+
+ URI baseURI;
+ try {
+ baseURI = new URI(
+ uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(),
+ "/", null, null);
+ } catch (URISyntaxException ex) {
+ LOG.error("Unable to determine base URI for " + uri);
+ return false;
+ }
+
+ NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
+ try {
+ nrc.parse(baseURI);
+ } catch (NoRobotException ex) {
+ LOG.error("Failure parsing robots.txt: " + ex.getMessage());
+ return false;
+ }
+ boolean test = nrc.isUrlAllowed(uri);
+ if (LOG.isInfoEnabled()) {
+ LOG.info(uri + " is " + (test ? "allowed" : "denied"));
+ }
+ return test;
+ }
+
+ public String getUserAgent() {
+ return userAgent;
+ }
+
+ public void setUserAgent(String userAgent) {
+ this.userAgent = userAgent;
+ this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
+ }
+
+ /**
+ * You can force that a site is allowed (ignoring the robots.txt). This should
+ * only be used on server that you control and where you have the permission
+ * to ignore the robots.txt.
+ *
+ * @return <code>true</code> if you are rude and ignore robots.txt.
+ * <code>false</code> if you are playing nice.
+ */
+ public boolean isForceAllow() {
+ return forceAllow;
+ }
+
+ /**
+ * You can force that a site is allowed (ignoring the robot.txt). This should
+ * only be used on server that you control and where you have the permission
+ * to ignore the robots.txt.
+ *
+ * @param forceAllow if you want to force an allow and ignore the robot.txt set
+ * to <code>true</code>. If you want to obey the rules and
+ * be polite set to <code>false</code>.
+ */
+ public void setForceAllow(boolean forceAllow) {
+ this.forceAllow = forceAllow;
+ }
+
+ protected HttpClient getHttpClient() {
+ return this.httpclient;
+ }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java Tue Dec 18 08:47:39 2012
@@ -22,20 +22,16 @@ import org.apache.http.message.BasicHead
import org.apache.http.message.BasicLineParser;
import org.apache.http.util.CharArrayBuffer;
-class LenientHttpResponseParser extends BasicLineParser
-{
+class LenientHttpResponseParser extends BasicLineParser {
- @Override
- public Header parseHeader(CharArrayBuffer buffer) throws ParseException
- {
- try
- {
- return super.parseHeader(buffer);
- } catch (ParseException ex)
- {
- // Suppress ParseException exception
- return new BasicHeader(buffer.toString(), null);
+ @Override
+ public Header parseHeader(CharArrayBuffer buffer) throws ParseException {
+ try {
+ return super.parseHeader(buffer);
+ } catch (ParseException ex) {
+ // Suppress ParseException exception
+ return new BasicHeader(buffer.toString(), null);
+ }
}
- }
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java Tue Dec 18 08:47:39 2012
@@ -27,28 +27,24 @@ import org.apache.http.auth.MalformedCha
import org.apache.http.client.AuthenticationHandler;
import org.apache.http.protocol.HttpContext;
-class NoAuthHandler implements AuthenticationHandler
-{
+class NoAuthHandler implements AuthenticationHandler {
- @Override
- public Map<String, Header> getChallenges(HttpResponse response, HttpContext context)
- throws MalformedChallengeException
- {
- return Collections.emptyMap();
- }
+ @Override
+ public Map<String, Header> getChallenges(HttpResponse response, HttpContext context)
+ throws MalformedChallengeException {
+ return Collections.emptyMap();
+ }
- @Override
- public boolean isAuthenticationRequested(HttpResponse response, HttpContext context)
- {
- return false;
- }
+ @Override
+ public boolean isAuthenticationRequested(HttpResponse response, HttpContext context) {
+ return false;
+ }
+
+ @Override
+ public AuthScheme selectScheme(Map<String, Header> challenges, HttpResponse response, HttpContext context)
+ throws AuthenticationException {
+ throw new AuthenticationException(
+ "Unable to respond to any of these challenges: " + challenges);
+ }
- @Override
- public AuthScheme selectScheme(Map<String, Header> challenges, HttpResponse response, HttpContext context)
- throws AuthenticationException
- {
- throw new AuthenticationException(
- "Unable to respond to any of these challenges: " + challenges);
- }
-
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java Tue Dec 18 08:47:39 2012
@@ -7,32 +7,28 @@ import junit.framework.TestSuite;
/**
* Unit test for simple App.
*/
-public class AppTest
- extends TestCase
-{
+public class AppTest
+ extends TestCase {
/**
* Create the test case
*
* @param testName name of the test case
*/
- public AppTest( String testName )
- {
- super( testName );
+ public AppTest(String testName) {
+ super(testName);
}
/**
* @return the suite of tests being tested
*/
- public static Test suite()
- {
- return new TestSuite( AppTest.class );
+ public static Test suite() {
+ return new TestSuite(AppTest.class);
}
/**
* Rigourous Test :-)
*/
- public void testApp()
- {
- assertTrue( true );
+ public void testApp() {
+ assertTrue(true);
}
}
Modified: incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml Tue Dec 18 08:47:39 2012
@@ -25,56 +25,57 @@
<http://www.apache.org />.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>droids</artifactId>
- <groupId>org.apache.droids</groupId>
- <version>0.3.0-incubating-SNAPSHOT</version>
- </parent>
- <artifactId>droids-examples</artifactId>
- <name>Apache Droids Examples</name>
- <inceptionYear>2007</inceptionYear>
- <description>
- Apache Droids Examples
- </description>
- <packaging>jar</packaging>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>droids</artifactId>
+ <groupId>org.apache.droids</groupId>
+ <version>0.3.0-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>droids-examples</artifactId>
+ <name>Apache Droids Examples</name>
+ <inceptionYear>2007</inceptionYear>
+ <description>
+ Apache Droids Examples
+ </description>
+ <packaging>jar</packaging>
- <dependencies>
- <dependency>
- <groupId>org.apache.droids</groupId>
- <artifactId>droids-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.droids</groupId>
- <artifactId>droids-tika</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.8.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.droids</groupId>
- <artifactId>droids-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>1.6.4</version>
- </dependency>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-tika</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.6.4</version>
+ </dependency>
- </dependencies>
+ </dependencies>
</project>