You are viewing a plain text version of this content. The canonical link for it is here.
Posted to droids-commits@incubator.apache.org by ja...@apache.org on 2010/05/09 11:16:51 UTC
svn commit: r942525 - in /incubator/droids/trunk: droids-core/
droids-spring/ droids-spring/src/test/java/org/apache/droids/dynamic/
droids-spring/src/test/resources/
Author: javier
Date: Sun May 9 11:16:51 2010
New Revision: 942525
URL: http://svn.apache.org/viewvc?rev=942525&view=rev
Log:
Test clases for Spring implementation.
Added:
incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java
incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
incubator/droids/trunk/droids-spring/src/test/resources/
incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml
incubator/droids/trunk/droids-spring/src/test/resources/droids-core.properties
Modified:
incubator/droids/trunk/droids-core/pom.xml
incubator/droids/trunk/droids-spring/pom.xml
Modified: incubator/droids/trunk/droids-core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/pom.xml?rev=942525&r1=942524&r2=942525&view=diff
==============================================================================
--- incubator/droids/trunk/droids-core/pom.xml (original)
+++ incubator/droids/trunk/droids-core/pom.xml Sun May 9 11:16:51 2010
@@ -40,6 +40,23 @@
Droids API and core components
</description>
<packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <!-- export our test classes in a separate jar -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
<dependencies>
<dependency>
Modified: incubator/droids/trunk/droids-spring/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/pom.xml?rev=942525&r1=942524&r2=942525&view=diff
==============================================================================
--- incubator/droids/trunk/droids-spring/pom.xml (original)
+++ incubator/droids/trunk/droids-spring/pom.xml Sun May 9 11:16:51 2010
@@ -53,6 +53,13 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-core</artifactId>
+ <version>${pom.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
<build>
<plugins>
Added: incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java?rev=942525&view=auto
==============================================================================
--- incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java (added)
+++ incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java Sun May 9 11:16:51 2010
@@ -0,0 +1,35 @@
+package org.apache.droids.dynamic;
+
+
+import org.apache.droids.helper.factories.HandlerFactory;
+import org.apache.droids.helper.factories.ParserFactory;
+import org.apache.droids.helper.factories.ProtocolFactory;
+import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+public class DroidsConfigTest {
+
+ private final static ApplicationContext context = new ClassPathXmlApplicationContext(
+ "classpath:/droids-core-test-context.xml");
+
+ private DroidsConfig droidsConfig = null;
+
+ @Before
+ public void setUp() throws Exception {
+ this.droidsConfig = (DroidsConfig) DroidsConfigTest.context.getBean("org.apache.droids.dynamic.DroidsConfig");
+ }
+
+ @Test
+ public void FactoryTest() {
+ Assert.assertEquals(URLFiltersFactory.class, droidsConfig.getFiltersFactory().getClass());
+ Assert.assertEquals(HandlerFactory.class, this.droidsConfig.getHandlerFactory().getClass());
+ Assert.assertEquals(ParserFactory.class, this.droidsConfig.getParserFactory().getClass());
+ Assert.assertEquals(ProtocolFactory.class, this.droidsConfig.getProtocolFactory().getClass());
+ }
+
+
+}
Added: incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java?rev=942525&view=auto
==============================================================================
--- incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java (added)
+++ incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java Sun May 9 11:16:51 2010
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.dynamic;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.Assert;
+
+import org.apache.droids.DroidsFactory;
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.TaskExceptionHandler;
+import org.apache.droids.api.TaskExceptionResult;
+import org.apache.droids.examples.handler.ReportHandler;
+import org.apache.droids.impl.ReportCrawlingDroid;
+import org.apache.droids.impl.SequentialTaskMaster;
+import org.apache.droids.localserver.LocalHttpServer;
+import org.apache.droids.localserver.ResourceHandler;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+public class TestSimpleDroid {
+
+ protected LocalHttpServer testserver;
+
+ private final static ApplicationContext context = new ClassPathXmlApplicationContext(
+ "classpath:/droids-core-test-context.xml");
+
+ private DroidsConfig droidsConfig = null;
+
+ @Before
+ public void setUp() throws Exception {
+ this.droidsConfig = (DroidsConfig) TestSimpleDroid.context
+ .getBean("org.apache.droids.dynamic.DroidsConfig");
+ this.testserver = new LocalHttpServer();
+ }
+
+ @Test
+ public void testReportCrawlingDroid() throws Exception {
+ this.testserver.register("*", new ResourceHandler());
+ this.testserver.start();
+
+ String baseURI = "http:/" + this.testserver.getServiceAddress();
+ String targetURI = baseURI + "/start_html";
+
+ Droid<Link> droid = createSimpleReportCrawlingDroid(targetURI);
+
+ droid.init();
+ droid.start();
+ droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
+
+ Assert.assertFalse(ReportHandler.getReport().isEmpty());
+ Assert.assertEquals(5, ReportHandler.getReport().size());
+ Assert.assertTrue(ReportHandler.getReport().contains(
+ baseURI + "/start_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(
+ baseURI + "/page1_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(
+ baseURI + "/page2_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(
+ baseURI + "/page3_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(
+ baseURI + "/page4_html"));
+
+ ReportHandler.recycle();
+ }
+
+ private Droid<Link> createSimpleReportCrawlingDroid(final String targetURI) {
+ Droid<Link> droid = this.droidsConfig.getDroid("report");
+
+ Assert.assertFalse("Droid is null.", droid == null);
+ Assert.assertTrue(
+ "The test droid must be an instance of ReportCrawlingDroid",
+ droid instanceof ReportCrawlingDroid);
+
+ final List<String> locations = new ArrayList<String>();
+ locations.add(targetURI);
+ ((CrawlingDroid) droid).setInitialLocations(locations);
+
+ return droid;
+ }
+
+}
Added: incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml?rev=942525&view=auto
==============================================================================
--- incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml (added)
+++ incubator/droids/trunk/droids-spring/src/test/resources/droids-core-test-context.xml Sun May 9 11:16:51 2010
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <!--
+ Using your own context
+ +++++++++++++++++++++++++
+ The easiest way is to
+ a) create a droids-your-context.xml
+ b) add:
+ <import resource="classpath:/org/apache/droids/dynamic/droids-core-context.xml"/>
+ c) implement your own beans which will override the imported ones
+ d) Call the ant target like:
+ ant droids.crawl default -Ddroids.spring.context=PATH/droids-your-context.xml
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:configurator="http://cocoon.apache.org/schema/configurator"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+ http://cocoon.apache.org/schema/configurator http://cocoon.apache.org/schema/configurator/cocoon-configurator-1.1.0.xsd">
+
+ <import resource="classpath:/org/apache/droids/dynamic/droids-core-context.xml"/>
+
+ <!-- configuration properties file -->
+ <bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+ <property name="locations" value="classpath:/droids-core.properties"/>
+ </bean>
+
+ <bean name="taskExceptionHandler"
+ class="org.apache.droids.impl.DefaultTaskExceptionHandler">
+ </bean>
+
+ <bean name="taskMaster"
+ class="org.apache.droids.impl.SequentialTaskMaster">
+ <property name="exceptionHandler" ref="taskExceptionHandler" />
+ <property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
+ <!--<property name="maxThreads" value="${droids.maxThreads}"/>-->
+ </bean>
+
+ <!-- Droids -->
+ <bean name="org.apache.droids.api.Droid/report"
+ class="org.apache.droids.impl.ReportCrawlingDroid">
+ <constructor-arg ref="taskMaster" />
+ <constructor-arg ref="org.apache.droids.impl.SimpleTaskQueue" />
+
+ <property name="protocolFactory" ref="org.apache.droids.helper.factories.ProtocolFactory"/>
+ <property name="parserFactory" ref="org.apache.droids.helper.factories.ParserFactory"/>
+ <property name="filtersFactory" ref="org.apache.droids.helper.factories.URLFiltersFactory"/>
+ </bean>
+ <!-- Queue -->
+ <bean id="org.apache.droids.impl.SimpleTaskQueue"
+ class="org.apache.droids.impl.SimpleTaskQueue">
+ <property name="maxSize" value="${droids.queue.maxSize}"/>
+ </bean>
+ <!-- Protocol -->
+ <bean
+ name="org.apache.droids.api.Protocol/http"
+ class="org.apache.droids.protocol.http.HttpProtocol" scope="singleton">
+ <property name="userAgent" value="DROIDS-crawler-x-m01y08"/>
+ <property name="forceAllow" value="${droids.protocol.http.force}"/>
+ </bean>
+ <bean name="org.apache.droids.api.Protocol/file"
+ class="org.apache.droids.protocol.file.FileProtocol" scope="singleton"/>
+ <!-- Parser -->
+ <bean
+ name="text/html"
+ class="org.apache.droids.parse.html.HtmlParser">
+ <property name="elements">
+ <map>
+ <entry key="a" value="href"/>
+ <entry key="link" value="href"/>
+ <entry key="img" value="src"/>
+ <entry key="script" value="src"/>
+ </map>
+ </property>
+ </bean>
+ <!-- Filter -->
+ <bean
+ name="org.apache.droids.api.URLFilter/org.apache.droids.net.RegexURLFilter"
+ class="org.apache.droids.net.RegexURLFilter">
+ <property name="file" value="${droids.filter.regex}"/>
+ </bean>
+ <!-- Handler -->
+ <bean
+ name="org.apache.droids.api.Handler/org.apache.droids.handle.Sysout"
+ class="org.apache.droids.handle.Sysout"/>
+ <bean
+ name="org.apache.droids.api.Handler/org.apache.droids.handle.Save"
+ class="org.apache.droids.handle.Save">
+ <property name="outputDir" value="${droids.handler.save.dir}"/>
+ <property name="includeHost" value="${droids.handler.save.includeHost}"/>
+ </bean>
+
+ <bean
+ name="org.apache.droids.delay.SimpleDelayTimer"
+ class="org.apache.droids.delay.SimpleDelayTimer">
+ <property name="delayMillis" value="${droids.delay.request}"/>
+ </bean>
+</beans>
Added: incubator/droids/trunk/droids-spring/src/test/resources/droids-core.properties
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-spring/src/test/resources/droids-core.properties?rev=942525&view=auto
==============================================================================
--- incubator/droids/trunk/droids-spring/src/test/resources/droids-core.properties (added)
+++ incubator/droids/trunk/droids-spring/src/test/resources/droids-core.properties Sun May 9 11:16:51 2010
@@ -0,0 +1,7 @@
+droids.maxThreads=5
+droids.queue.maxSize=-1
+droids.protocol.http.force=true
+droids.filter.regex=classpath:/regex-urlfilter.txt
+droids.handler.save.dir=tmp/
+droids.handler.save.includeHost=true
+droids.delay.request=100
\ No newline at end of file
Re: svn commit: r942525 - in /incubator/droids/trunk: droids-core/
droids-spring/ droids-spring/src/test/java/org/apache/droids/dynamic/
droids-spring/src/test/resources/
Posted by Javier Puerto <ja...@apache.org>.
2010/5/10 Thorsten Scherler <th...@apache.org>
>
> On 09/05/2010, at 13:16, javier@apache.org wrote:
>
> > +++
> incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java
> Sun May 9 11:16:51 2010
> > @@ -0,0 +1,35 @@
> > +package org.apache.droids.dynamic;
> > +
>
>
> Hi Javier,
>
Hi Thorsten,
>
> you forgot the license header.
>
You are right, I forget to add the license header. I will commit the new
file with license later.
>
> salu2
>
salu2
>
> Thorsten Scherler <thorsten.at.apache.org>
> Open Source Java <consulting, training and solutions>
>
>
Re: svn commit: r942525 - in /incubator/droids/trunk: droids-core/ droids-spring/ droids-spring/src/test/java/org/apache/droids/dynamic/ droids-spring/src/test/resources/
Posted by Thorsten Scherler <th...@apache.org>.
On 09/05/2010, at 13:16, javier@apache.org wrote:
> +++ incubator/droids/trunk/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java Sun May 9 11:16:51 2010
> @@ -0,0 +1,35 @@
> +package org.apache.droids.dynamic;
> +
Hi Javier,
you forgot the license header.
salu2
Thorsten Scherler <thorsten.at.apache.org>
Open Source Java <consulting, training and solutions>