You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by mi...@apache.org on 2009/09/01 22:11:33 UTC
svn commit: r810273 [4/4] - in /incubator/droids/trunk/droids-crawler: ./
docs/ docs/diagrams/ src/ src/main/ src/main/groovy/ src/main/java/
src/main/java/org/ src/main/java/org/apache/
src/main/java/org/apache/droids/ src/main/java/org/apache/droids/...
Added: incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java (added)
+++ incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.droids.crawler.util;
+
+/**
+ * For the filter framework, a filter with a higher weight (i.e. more important) shall be executed first.
+ */
+public interface Weighted {
+ int getWeight();
+}
Added: incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java (added)
+++ incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.util.logging;
+
+import java.util.Map;
+import java.util.Collection;
+
+public class LogUtils{
+
+ /**
+ * convert any object to String in a way similar to Groovy syntax.
+ * e.g. any string -> "any string"
+ * map of key and value -> [key:value]
+ * collection of v1, v2 -> [v1, v2]
+ */
+ public static String toString(Object target){
+ return append(target, new StringBuilder()).toString();
+ }
+
+ /**
+ * This method creates a String representation of any target in a way similar to Groovy syntax
+ */
+ public static StringBuilder append(Object target, StringBuilder out){
+ if (target instanceof String){
+ out.append("\"").append(target).append("\"");
+ } else if (target instanceof Boolean || target instanceof Integer){
+ out.append(target);
+ } else if (target instanceof Map){
+ Map map = (Map) target;
+ boolean first = true;
+ out.append("[");
+ if (map.size() == 0){
+ out.append(":");
+ } else{
+ for (Object key : map.keySet()){
+ if (!first) out.append(", ");
+ append(key, out);
+ out.append(":");
+ append(map.get(key), out);
+ first = false;
+ }
+ }
+ out.append("]");
+ } else if (target instanceof Collection){
+ boolean first = true;
+ out.append("[");
+ for (Object t : (Collection) target){
+ if (!first) out.append(",");
+ append(t, out);
+ first = false;
+ }
+ out.append("]");
+ } else{
+ out.append(target != null ? target.toString() : "null");
+ }
+ return out;
+ }
+
+}
Added: incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration debug="false" xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <!-- http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html -->
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%-40.40c] [%-11t] - %m%n"/>
+ <!--<param name="ConversionPattern" value="%d %-5p [%-30.40c{1}] [%t] - %m%n"/>-->
+ </layout>
+ </appender>
+
+ <logger name="org.apache.http.wire" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="CONSOLE"/>
+ </logger>
+
+ <logger name="org.apache.droids.crawler" additivity="false">
+ <level value="debug"/>
+ <appender-ref ref="CONSOLE"/>
+ </logger>
+
+ <root>
+ <priority value="info"/>
+ <appender-ref ref="CONSOLE"/>
+ </root>
+
+</log4j:configuration>
Added: incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <context:annotation-config/>
+
+ <!-- for multiple remote crawler service -->
+ <bean id="crawlerService" class="org.apache.droids.crawler.RoundRobinCrawlerService">
+ <property name="crawlerServices">
+ <util:list>
+ <bean id="remoteCrawlerService0"
+ class="org.springframework.remoting.httpinvoker.HttpInvokerProxyFactoryBean">
+ <!-- TODO: change the following serviceUrl -->
+ <property name="serviceUrl" value="http://YOUR_APP0.appspot.com"/>
+ <property name="serviceInterface" value="org.apache.droids.crawler.CrawlerService"/>
+ </bean>
+ <bean id="remoteCrawlerService1"
+ class="org.springframework.remoting.httpinvoker.HttpInvokerProxyFactoryBean">
+ <!-- TODO: change the following serviceUrl -->
+ <property name="serviceUrl" value="http://YOUR_APP1.appspot.com"/>
+ <property name="serviceInterface" value="org.apache.droids.crawler.CrawlerService"/>
+ </bean>
+ </util:list>
+ </property>
+ </bean>
+
+ <!-- for single remote crawler service -->
+ <!--
+ <bean id="crawlerService" class="org.springframework.remoting.httpinvoker.HttpInvokerProxyFactoryBean">
+ <property name="serviceUrl" value="http://YOUR_APP0.appspot.com"/>
+ <property name="serviceInterface" value="org.apache.droids.crawler.CrawlerService"/>
+ </bean>
+ -->
+
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.junit.Test;
+import static org.junit.Assert.assertNotNull;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+public class DependencyTest{
+ @Test public void testSpring(){
+ assertNotNull(new ClassPathXmlApplicationContext("test-fetcher.HttpFetcherTest.xml"));
+ //assertNotNull(new ClassPathXmlApplicationContext("test-fetcher.HttpFetcherTest.xml").getBean("httpClient"));
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.parser.ParserException;
+import org.apache.droids.crawler.extractor.ExtractorException;
+
+/**
+ * This is for unit testing only
+ */
+public class DummyCrawlerService implements CrawlerService<Link>{
+ protected int fetchCount, parseCount, extractCount;
+
+ public String getVersion(){
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public String getNode(){
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public Link fetch(Link link) throws FetcherException{
+ fetchCount++;
+ return link;
+ }
+
+ public Link parse(Link link) throws ParserException{
+ parseCount++;
+ return link;
+ }
+
+ public Link extract(Link link) throws ExtractorException{
+ extractCount++;
+ return link;
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+
+public class LinkTest{
+
+ @Test public void testType(){
+ assertTrue(new Link() instanceof Map);
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.fetcher.Fetcher;
+import org.apache.droids.crawler.parser.ParserException;
+import org.apache.droids.crawler.parser.Parser;
+import org.apache.droids.crawler.extractor.ExtractorException;
+
+import java.net.URISyntaxException;
+import java.util.Set;
+import java.util.Map;
+
+public class LocalCrawlerServiceTest{
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-LocalCrawlerServiceTest.xml");
+ static String url = "http://www.apache.org";
+
+ @Test public void testFetch() throws URISyntaxException, FetcherException{
+ CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+ assertNotNull(crawlerService);
+ Link fetchedLink = crawlerService.fetch(new Link(url));
+ assertNotNull(fetchedLink);
+ assertNotNull(fetchedLink.get("fetched"));
+ assertTrue(fetchedLink.get("fetched") instanceof Fetcher);
+ assertNotNull(fetchedLink.get("fetched", Fetcher.class).getEntity());
+ }
+
+ @Test public void testFetchParse() throws URISyntaxException, FetcherException, ParserException{
+ CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+ Link fetchedLink = crawlerService.fetch(new Link(url));
+ Link parsedLink = crawlerService.parse(fetchedLink);
+ assertTrue(parsedLink.get("parsed") instanceof Parser);
+ assertNotNull(parsedLink.get("parsed", Parser.class).getData());
+ }
+
+ @Test public void testParseWithoutFetch() throws URISyntaxException, FetcherException, ParserException{
+ CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+ Link parsedLink = crawlerService.parse(new Link(url));
+ assertTrue(parsedLink.get("parsed") instanceof Parser);
+ Map<String, Map<String, Set<String>>> data = (Map<String, Map<String, Set<String>>>) parsedLink.get("parsed", Parser.class).getData();
+ assertNotNull(data);
+ assertEquals(3, data.size());
+ }
+
+ @Test public void testExtractWithoutFetchAndParse() throws ExtractorException, URISyntaxException{
+ CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+ Link link = crawlerService.extract(new Link(url));
+ assertFalse(link.containsKey("parsed"));
+ assertTrue(link.containsKey("extracted"));
+ assertTrue(link.get("extracted") instanceof Set);
+ Set<Link> outlinks = link.get("extracted", Set.class);
+ System.out.println(outlinks);
+ assertTrue(outlinks.size() > 100);
+ }
+
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.context.ApplicationContext;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.apache.droids.crawler.fetcher.FetcherException;
+
+import java.net.URISyntaxException;
+
+public class RoundRobinCrawlerServiceTest{
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-RoundRobinCrawlerServiceTest.xml");
+
+ @Test public void testFetch() throws FetcherException, URISyntaxException{
+ RoundRobinCrawlerService service = context.getBean("crawlerService", RoundRobinCrawlerService.class);
+ assertEquals(3, service.getCrawlerServices().size());
+ for (int i = 0; i < 30; i++){
+ service.fetch(new Link("http://www.apache.org/page/" + i));
+ }
+
+ for (Object delegate : service.getCrawlerServices()){
+ DummyCrawlerService dummyDelegate = (DummyCrawlerService) delegate;
+ assertEquals(10, dummyDelegate.fetchCount);
+ }
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.junit.Test;
+import static org.junit.Assert.assertNotNull;
+import org.apache.droids.exception.DroidsException;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.context.ApplicationContext;
+
+public class SimpleCrawlerControllerTest{
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-StandaloneCrawlerControllerTest.xml");
+
+ @Test public void testCrawlerControllerInit() throws DroidsException{
+ StandaloneCrawlerController controller = new StandaloneCrawlerController();
+ controller.init();
+ assertNotNull(controller.getQueue());
+ }
+
+ @Test public void testCrawlerControllerStart() throws CrawlerException{
+ CrawlerController controller = context.getBean("crawlerController", CrawlerController.class);
+ controller.start();
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.extractor;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.apache.droids.crawler.Link;
+import org.apache.droids.crawler.filter.extract.IncludeFilter;
+import org.apache.droids.crawler.filter.extract.RefererFilter;
+import org.apache.droids.crawler.util.WeightComparator;
+import org.apache.droids.crawler.fetcher.HttpFetcher;
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.parser.Parser;
+import org.apache.droids.crawler.parser.ParserException;
+import org.apache.droids.crawler.parser.impl.NekoHtmlParser;
+import org.apache.droids.crawler.parser.impl.AbstractHierarchicalDataParser;
+import org.apache.droids.core.Entity;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.regex.Matcher;
+import java.net.URISyntaxException;
+
+public class HtmlElementLinkExtractorTest{
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-extractor.HtmlElementLinkExtractorTest.xml");
+
+ @Test public void testParser() throws URISyntaxException, ParserException, FetcherException{
+ Link link = new Link("http://www.apache.org");
+ Parser parser = context.getBean("nekoHtmlParser0", NekoHtmlParser.class);
+ parser.parse(link, new HttpFetcher().fetch(link).getEntity(), null);
+ Set links = context.getBean("htmlElementLinkExtractor", HtmlElementLinkExtractor.class).extract(link, parser);
+ assertNotNull(links);
+ assertTrue(links.size() > 100);
+ }
+
+
+ @Test public void testExtractStyleBackground() throws URISyntaxException{
+ Link link = new Link("http://www.w3schools.com/css/tryit_view.asp?filename=trycss_background-image");
+ final Map data = new HashMap(), style = new HashMap();
+ Set blank = new HashSet();
+ data.put("STYLE", style);
+ style.put("", blank);
+ blank.add("body {background-image:url('paper.gif')}");
+
+ Set<Link> links = context.getBean("htmlElementLinkExtractor", Extractor.class).extract(link, new AbstractHierarchicalDataParser<Link>(){
+ public Parser parse(Link link, Entity entity, Map<String, Object> params) throws ParserException{return null; }
+
+ @Override public Map getData(){return data;}
+
+ public void reset(){}
+
+ public boolean matches(Link link){return false;}
+ });
+ assertEquals(1, links.size());
+ assertEquals("http://www.w3schools.com/css/paper.gif", links.iterator().next().getUrl());
+ }
+
+ @Test public void testExtractFilterSorting(){ //TODO move to another test class
+
+ IncludeFilter includeFilter = new IncludeFilter("");
+ includeFilter.setWeight(100);
+ RefererFilter refererFilter = new RefererFilter();
+ refererFilter.setWeight(200);
+
+ HtmlElementLinkExtractor extractor = new HtmlElementLinkExtractor();
+ extractor.addFilter(includeFilter);
+ extractor.addFilter(refererFilter);
+ extractor.setFilterComparator(new WeightComparator());
+ extractor.init();
+
+ assertTrue(extractor.filters.get(0) instanceof RefererFilter);
+
+ }
+
+ @Test public void testCSSBackground(){
+ Matcher matcher = null;
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url('paper.gif')}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url(\"paper.gif\")}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url(paper.gif)}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image :url(paper.gif)}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image: url(paper.gif)}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url( 'paper.gif' )}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+
+ matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url( paper.gif )}");
+ assertTrue(matcher.matches());
+ assertEquals("paper.gif", matcher.group(1));
+ }
+
+
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.fetcher;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import org.apache.droids.crawler.Link;
+import org.apache.commons.io.IOUtils;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.net.URISyntaxException;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+//TODO find a better way to test than making http request to apache
+public class HttpFetcherTest{
+
+ @Test public void testDefaultFetch() throws FetcherException, URISyntaxException, IOException{
+ Fetcher fetcher = new HttpFetcher().fetch(new Link("http://incubator.apache.org/droids/"));
+ assertEquals(200, fetcher.getStatusCode());
+ assertNotNull(fetcher.getEntity());
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copy(fetcher.getEntity().getContent(), out);
+ assertTrue(out.toByteArray().length > 15000);
+ }
+
+ @Test public void testDefaultFetch_Spring() throws FetcherException, URISyntaxException{
+ Fetcher fetcher = new ClassPathXmlApplicationContext("test-fetcher.HttpFetcherTest.xml").getBean("httpFetcher", HttpFetcher.class);
+ fetcher.fetch(new Link("http://incubator.apache.org/droids/"), null);
+ assertEquals(200, fetcher.getStatusCode());
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.fetcher.appengine;
+
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import org.apache.droids.crawler.Link;
+
+import java.net.URISyntaxException;
+
+public class AppEngineFetcherTest{
+
+ @Test public void testMatch() throws URISyntaxException{
+ assertTrue(new AppEngineFetcher().matches(new Link("http://www.apache.org")));
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.filter;
+
+import org.apache.droids.crawler.Link;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+
+public class StatsFilterTest{
+
+ @Test public void testPollCount() throws URISyntaxException{
+ StatsFilter filter = new StatsFilter().init();
+
+ for (int i = 0; i < 10; i++) filter.polled(new Link("http://www.apache.org/page/" + i));
+
+ Map report = filter.report();
+ assertEquals(9, report.size());
+ assertEquals(10, report.get("poll.count"));
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,34 @@
+package org.apache.droids.crawler.parser;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.apache.droids.crawler.Link;
+import org.apache.droids.crawler.parser.impl.NekoHtmlParser;
+import static junit.framework.Assert.assertEquals;
+
+import java.net.URISyntaxException;
+
+public class DefaultParserFactoryTest{
+
+ @Test public void testAutowiringOfParsers(){
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-parser.DefaultParserFactoryTest.xml");
+ DefaultParserFactory factory = context.getBean("parserFactory", DefaultParserFactory.class);
+ assertNotNull(factory);
+ assertNotNull(factory.getParsers());
+ assertEquals(3, factory.getParsers().size());
+ }
+
+ @Test public void testTextHtml() throws URISyntaxException{
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-parser.DefaultParserFactoryTest.xml");
+ DefaultParserFactory factory = context.getBean("parserFactory", DefaultParserFactory.class);
+ Link link = new Link("http://www.apache.org");
+ link.put("contentType", "text/html");
+ Parser parser = factory.newParser(link);
+ assertNotNull(parser);
+ assertTrue(parser instanceof NekoHtmlParser);
+ }
+
+}
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,52 @@
+package org.apache.droids.crawler.parser.impl;
+
+import org.apache.droids.core.Entity;
+import org.apache.droids.crawler.Link;
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.fetcher.HttpFetcher;
+import org.apache.droids.crawler.parser.Parser;
+import org.apache.droids.crawler.parser.ParserException;
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Set;
+
+
+public class NekoHtmlParserTest{
+ /*ApplicationContext context;
+
+ @Before public void setUp(){
+ context = ;
+ }*/
+
+ @Test public void testParser_A_href() throws URISyntaxException, FetcherException, ParserException{
+ Link link = new Link("http://www.apache.org/");
+ Entity entity = new HttpFetcher().fetch(link).getEntity();
+ Parser parser = new ClassPathXmlApplicationContext("test-parser.impl.NekoHtmlParserTest.xml").getBean("nekoHtmlParser0", NekoHtmlParser.class);
+ Map<String, Map<String, Set<Link>>> result = (Map<String, Map<String, Set<Link>>>) parser.parse(link, entity, null).getData();
+ assertNotNull(result);
+ assertEquals(1, result.size());
+ assertEquals("A", result.keySet().iterator().next());
+ assertEquals(1, result.get("A").size());
+ assertEquals("href", result.get("A").keySet().iterator().next());
+ assertTrue(result.get("A").get("href").size() > 100);
+ }
+
+
+ @Test public void testParser_Style() throws URISyntaxException, FetcherException, ParserException{
+ Link link = new Link("http://www.w3schools.com/css/tryit_view.asp?filename=trycss_background-image");
+ Entity entity = new HttpFetcher().fetch(link).getEntity();
+ Parser parser = new ClassPathXmlApplicationContext("test-parser.impl.NekoHtmlParserTest.xml").getBean("nekoHtmlParser1", NekoHtmlParser.class);
+ Map<String, Map<String, Set<Link>>> result = (Map<String, Map<String, Set<Link>>>) parser.parse(link, entity, null).getData();
+ assertNotNull(result);
+ assertEquals(1, result.size());
+ assertEquals("STYLE", result.keySet().iterator().next());
+ assertEquals(1, result.get("STYLE").size());
+ assertEquals("", result.get("STYLE").keySet().iterator().next());
+ assertTrue(result.get("STYLE").get("").size() == 1);
+ assertEquals("body {background-image:url('paper.gif')}", result.get("STYLE").get("").toArray()[0]);
+ }
+}
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java Tue Sep 1 22:11:29 2009
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.util;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ParamUtilsTest{
+
+ @Test public void testResolve(){
+ ApplicationContext context = new ClassPathXmlApplicationContext("test-util.ParamUtilsTest.xml");
+ assertTrue(context.containsBean("map"));
+
+ Map cfg = new HashMap();
+ cfg.put("key", Map.class);
+ Map result = (Map) ParamUtils.resolve(cfg, "key", Map.class, context);
+ assertNotNull(result);
+ assertTrue(result.containsKey("k"));
+ assertEquals("v", result.get("k"));
+ }
+}
Added: incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration debug="false" xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <!-- http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html -->
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%-40.40c] [%-11t] [%t] - %m%n"/>
+ <!--<param name="ConversionPattern" value="%d %-5p [%-30.40c{1}] [%t] - %m%n"/>-->
+ </layout>
+ </appender>
+
+ <logger name="org.apache.http.wire" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="CONSOLE"/>
+ </logger>
+
+ <logger name="org.apache.droids.crawler.CrawlerWorker" additivity="false">
+ <level value="trace"/>
+ <appender-ref ref="CONSOLE"/>
+ </logger>
+
+ <logger name="org.apache.droids.crawler" additivity="false">
+ <level value="debug"/>
+ <appender-ref ref="CONSOLE"/>
+ </logger>
+
+ <root>
+ <priority value="info"/>
+ <appender-ref ref="CONSOLE"/>
+ </root>
+
+</log4j:configuration>
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <context:annotation-config/>
+ <!--<context:component-scan base-package="org.apache.droids.crawler"/>-->
+
+ <import resource="classpath:/test-fetcher.HttpFetcherTest.xml"/>
+
+ <bean class="org.apache.droids.crawler.filter.fetch.LinkAttributeFilter"/>
+
+ <bean id="nekoHtmlParser" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype"/>
+
+ <bean id="crawlerService" class="org.apache.droids.crawler.LocalCrawlerService">
+ <property name="fetcherFactory">
+ <bean class="org.apache.droids.crawler.fetcher.DefaultFetcherFactory">
+ <property name="fetchers">
+ <util:list>
+ <bean class="org.apache.droids.crawler.fetcher.HttpFetcher"/>
+ </util:list>
+ </property>
+ </bean>
+ </property>
+ <property name="parserFactory">
+ <bean class="org.apache.droids.crawler.parser.DefaultParserFactory">
+ <property name="parsers">
+ <util:list>
+ <bean class="org.apache.droids.crawler.parser.impl.NekoHtmlParser"/>
+ </util:list>
+ </property>
+ </bean>
+ </property>
+ <property name="extractors">
+ <util:list>
+ <bean class="org.apache.droids.crawler.extractor.HtmlElementLinkExtractor"/>
+ </util:list>
+ </property>
+ </bean>
+
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <context:annotation-config/>
+
+ <bean id="crawlerService" class="org.apache.droids.crawler.RoundRobinCrawlerService">
+ <property name="crawlerServices">
+ <util:list>
+ <bean id="service0" class="org.apache.droids.crawler.DummyCrawlerService"/>
+ <bean id="service1" class="org.apache.droids.crawler.DummyCrawlerService"/>
+ <bean id="service2" class="org.apache.droids.crawler.DummyCrawlerService"/>
+ </util:list>
+ </property>
+ </bean>
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <context:annotation-config/>
+ <!--<context:component-scan base-package="org.apache.droids.crawler"/>-->
+ <bean id="queue" class="org.apache.droids.crawler.util.DefaultLinkQueue"/>
+
+ <bean id="worker" class="org.apache.droids.crawler.CrawlerWorker" scope="prototype">
+ <property name="queue" ref="queue"/>
+ <property name="crawlerService" ref="crawlerService"/>
+ </bean>
+
+ <bean id="crawlerController" class="org.apache.droids.crawler.StandaloneCrawlerController" init-method="init">
+ <property name="executorService">
+ <bean class="org.apache.droids.crawler.util.CrawlerExecutorService">
+ <constructor-arg index="0" value="1"/>
+ <constructor-arg index="1">
+ <bean class="org.apache.droids.core.thread.DefaultThreadFactory"/>
+ </constructor-arg>
+ <constructor-arg index="2">
+ <bean class="org.apache.droids.core.thread.DefaultRejectedExecutionHandler"/>
+ </constructor-arg>
+ </bean>
+ </property>
+ <property name="seeds">
+ <util:list>
+ <value>http://incubator.apache.org/droids/</value>
+ </util:list>
+ </property>
+ <property name="queue" ref="queue"/>
+ </bean>
+
+
+ <bean id="httpFetcher" class="org.apache.droids.crawler.fetcher.HttpFetcher" scope="prototype"/>
+ <bean id="htmlParser" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype"/>
+ <bean id="htmlExtractor" class="org.apache.droids.crawler.extractor.HtmlElementLinkExtractor"/>
+
+ <bean id="crawlerService" class="org.apache.droids.crawler.LocalCrawlerService">
+ <property name="fetcherFactory">
+ <bean class="org.apache.droids.crawler.fetcher.DefaultFetcherFactory">
+ <property name="fetchers">
+ <util:list>
+ <ref bean="httpFetcher"/>
+ </util:list>
+ </property>
+ </bean>
+ </property>
+ <property name="parserFactory">
+ <bean class="org.apache.droids.crawler.parser.DefaultParserFactory">
+ <property name="parsers">
+ <util:list>
+ <ref bean="htmlParser"/>
+ </util:list>
+ </property>
+ </bean>
+ </property>
+ <property name="extractors">
+ <util:list>
+ <ref bean="htmlExtractor"/>
+ </util:list>
+ </property>
+ </bean>
+
+
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <import resource="classpath:/test-parser.impl.NekoHtmlParserTest.xml"/>
+
+ <bean id="htmlElementLinkExtractor" class="org.apache.droids.crawler.extractor.HtmlElementLinkExtractor"/>
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <!--<context:annotation-config/>-->
+ <!--<context:component-scan base-package="org.apache.droids.crawler"/>-->
+
+ <!-- this is an example of how to configure HttpClient in Spring -->
+ <bean id="httpParams" class="org.apache.http.params.BasicHttpParams"/>
+ <bean class="org.apache.http.params.HttpConnectionParamBean">
+ <constructor-arg index="0">
+ <ref bean="httpParams"/>
+ </constructor-arg>
+ <property name="connectionTimeout" value="60000"/>
+ <property name="tcpNoDelay" value="false"/>
+ <property name="staleCheckingEnabled" value="false"/>
+ <property name="socketBufferSize" value="8192"/>
+ </bean>
+ <bean class="org.apache.http.params.HttpProtocolParamBean">
+ <constructor-arg index="0">
+ <ref bean="httpParams"/>
+ </constructor-arg>
+ <property name="contentCharset" value="ISO-8859-1"/>
+ <!--<property name="httpElementCharset" value="UTF-8"/>-->
+ <property name="useExpectContinue" value="true"/>
+ <property name="userAgent"
+ value="Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.1) Gecko/20090715 Firefox/3.0.10"/>
+ <!--<property name="version" value="1.1"/>-->
+ </bean>
+
+
+ <bean id="httpClient" class="org.apache.http.impl.client.DefaultHttpClient">
+ <constructor-arg index="0" ref="httpParams"/>
+ </bean>
+
+ <bean id="httpFetcher" class="org.apache.droids.crawler.fetcher.HttpFetcher" scope="prototype">
+ <property name="httpClient" ref="httpClient"/>
+ </bean>
+
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <context:annotation-config/>
+ <context:component-scan base-package="org.apache.droids.crawler"/>
+
+ <!-- no configuration of parsers to test autowiring -->
+ <bean id="parserFactory" class="org.apache.droids.crawler.parser.DefaultParserFactory"/>
+ <bean class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype"/>
+ <!--<bean class="org.apache.droids.crawler.parser.impl.JerichoHtmlParser" scope="prototype"/>-->
+ <bean class="org.apache.droids.crawler.parser.impl.SAXElementParser" scope="prototype"/>
+ <bean class="org.apache.droids.crawler.parser.impl.StAXElementParser" scope="prototype"/>
+
+
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <context:annotation-config/>
+ <context:component-scan base-package="org.apache.droids.crawler"/>
+
+ <!-- no configuration of parsers to test autowiring -->
+ <bean id="nekoHtmlParser0" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype">
+ <property name="elements">
+ <util:map>
+ <entry key="A">
+ <util:list>
+ <value>href</value>
+ </util:list>
+ </entry>
+ </util:map>
+ </property>
+ </bean>
+
+ <!-- no configuration of parsers to test autowiring -->
+ <bean id="nekoHtmlParser1" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype">
+ <property name="elements">
+ <util:map>
+ <entry key="STYLE">
+ <util:list>
+ <value></value>
+ </util:list>
+ </entry>
+ </util:map>
+ </property>
+ </bean>
+
+</beans>
\ No newline at end of file
Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml Tue Sep 1 22:11:29 2009
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+ <util:map id="map">
+ <entry key="k" value="v"/>
+ </util:map>
+
+</beans>
\ No newline at end of file