You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by ol...@apache.org on 2008/09/13 15:35:03 UTC
svn commit: r694942 -
/labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
Author: olegk
Date: Sat Sep 13 06:35:03 2008
New Revision: 694942
URL: http://svn.apache.org/viewvc?rev=694942&view=rev
Log:
Simple Droids runtime that wires various components together in Java code without using a DI framework
Added:
labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java (with props)
Added: labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java?rev=694942&view=auto
==============================================================================
--- labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java (added)
+++ labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java Sat Sep 13 06:35:03 2008
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.examples;
+
+import java.util.HashMap;
+
+import org.apache.droids.Core;
+import org.apache.droids.DefaultCrawler;
+import org.apache.droids.delay.SimpleDelayTimer;
+import org.apache.droids.exception.DroidsException;
+import org.apache.droids.handle.Sysout;
+import org.apache.droids.helper.factories.DroidFactory;
+import org.apache.droids.helper.factories.HandlerFactory;
+import org.apache.droids.helper.factories.ParserFactory;
+import org.apache.droids.helper.factories.ProtocolFactory;
+import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.net.RegexURLFilter;
+import org.apache.droids.parse.html.HtmlParser;
+import org.apache.droids.protocol.http.Http;
+import org.apache.droids.queue.Simple;
+
+/**
+ * Simple Droids runtime that wires various components together in Java code
+ * without using a DI framework.
+ *
+ */
+public class SimpleRuntime {
+
+ private SimpleRuntime(){
+ }
+
+ public static void main(String[] args) throws DroidsException {
+
+ if (args.length < 1) {
+ System.out.println("Please specify a URL to crawl");
+ System.exit(-1);
+ }
+ String targetURL = args[0];
+
+ // Create parser factory. Support basic HTML markup only
+ ParserFactory parserFactory = new ParserFactory();
+ HtmlParser htmlParser = new HtmlParser();
+ htmlParser.setElements(new HashMap<String, String>());
+ htmlParser.getElements().put("a", "href");
+ htmlParser.getElements().put("link", "href");
+ htmlParser.getElements().put("img", "src");
+ htmlParser.getElements().put("script", "src");
+ parserFactory.setMap(new HashMap<String, Object>());
+ parserFactory.getMap().put("text/html", htmlParser);
+
+ // Create protocol factory. Support HTTP only.
+ ProtocolFactory protocolFactory = new ProtocolFactory();
+ Http httpProtocol = new Http();
+ httpProtocol.setForceAllow(true);
+ protocolFactory.setMap(new HashMap<String, Object>());
+ protocolFactory.getMap().put("http", httpProtocol);
+
+ // Create URL filter factory.
+ URLFiltersFactory filtersFactory = new URLFiltersFactory();
+ RegexURLFilter defaultURLFilter = new RegexURLFilter();
+ defaultURLFilter.setFile("classpath:/regex-urlfilter.txt");
+ filtersFactory.setMap(new HashMap<String, Object>());
+ filtersFactory.getMap().put("default", defaultURLFilter);
+
+ // Create handler factory. Provide sysout handler only.
+ HandlerFactory handlerFactory = new HandlerFactory();
+ Sysout defaultHandler = new Sysout();
+ handlerFactory.setMap(new HashMap<String, Object>());
+ handlerFactory.getMap().put("default", defaultHandler);
+
+ // Create droid factory. Leave it empty for now.
+ DroidFactory droidFactory = new DroidFactory();
+ droidFactory.setMap(new HashMap<String, Object>());
+
+ // Assemble Droid Core
+ Core core = new Core();
+ core.setDroids(droidFactory);
+ core.setParserFactory(parserFactory);
+ core.setProtocolFactory(protocolFactory);
+ core.setFiltersFactory(filtersFactory);
+ core.setHandlerFactory(handlerFactory);
+
+ // Create default droid
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelay(100);
+
+ Simple simpleQueue = new Simple();
+ simpleQueue.setMaxDepth(3);
+ simpleQueue.setMaxSize(-1);
+
+ DefaultCrawler defaultCrawler = new DefaultCrawler();
+ defaultCrawler.setCore(core);
+ defaultCrawler.setDelayTimer(simpleDelayTimer);
+ defaultCrawler.setQueue(simpleQueue);
+ defaultCrawler.setMaxThreads(5);
+ defaultCrawler.setUrl(targetURL);
+
+ // Add default droid to the droid factory
+ droidFactory.getMap().put("default", defaultCrawler);
+
+ // Start
+ core.start("default");
+ }
+
+}
Propchange: labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Propchange: labs/droids/trunk/src/examples/java/org/apache/droids/examples/SimpleRuntime.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org