You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/08/01 14:23:56 UTC
svn commit: r1367952 - in /manifoldcf/trunk: ./
tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/
Author: kwright
Date: Wed Aug 1 12:23:55 2012
New Revision: 1367952
URL: http://svn.apache.org/viewvc?rev=1367952&view=rev
Log:
First attempt at a web crawl load test that uses hopcount filtering.
Added:
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java (with props)
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java (with props)
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java (with props)
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java (with props)
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java (with props)
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java (with props)
Modified:
manifoldcf/trunk/build.xml
Modified: manifoldcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/build.xml?rev=1367952&r1=1367951&r2=1367952&view=diff
==============================================================================
--- manifoldcf/trunk/build.xml (original)
+++ manifoldcf/trunk/build.xml Wed Aug 1 12:23:55 2012
@@ -2195,6 +2195,14 @@
<ant dir="tests/rss" target="run-load-HSQLDB"/>
</target>
+ <target name="run-webcrawler-loadtests-postgresql" depends="build-tests-framework,build-tests-webcrawler-connector,build-tests-nulloutput-connector,calculate-webcrawler-tests-condition" if="webcrawler-tests.include">
+ <ant dir="tests/webcrawler" target="run-load-postgresql"/>
+ </target>
+
+ <target name="run-webcrawler-loadtests-mysql" depends="build-tests-framework,build-tests-webcrawler-connector,build-tests-nulloutput-connector,calculate-webcrawler-tests-condition" if="webcrawler-tests.include">
+ <ant dir="tests/webcrawler" target="run-load-mysql"/>
+ </target>
+
<target name="run-webcrawler-UI-tests-derby" depends="build-tests-framework,build-tests-webcrawler-connector,build-tests-nulloutput-connector,calculate-webcrawler-tests-condition" if="webcrawler-tests.include">
<ant dir="tests/webcrawler" target="run-UI-derby"/>
</target>
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java?rev=1367952&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java Wed Aug 1 12:23:55 2012
@@ -0,0 +1,53 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseMySQL extends org.apache.manifoldcf.crawler.tests.BaseITMySQL
+{
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"Web Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.webcrawler.WebcrawlerConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java?rev=1367952&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java Wed Aug 1 12:23:55 2012
@@ -0,0 +1,53 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BasePostgresql extends org.apache.manifoldcf.crawler.tests.BaseITPostgresql
+{
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"Web Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.webcrawler.WebcrawlerConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BasePostgresql.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java?rev=1367952&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java Wed Aug 1 12:23:55 2012
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlMySQLLT extends BaseMySQL
+{
+
+ protected BigCrawlTester tester;
+ protected MockWebService webService = null;
+
+ public BigCrawlMySQLLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
+
+ // Setup and teardown the mock wiki service
+
+ @Before
+ public void createWebService()
+ throws Exception
+ {
+ webService = new MockWebService(10);
+ webService.start();
+ }
+
+ @After
+ public void shutdownWebService()
+ throws Exception
+ {
+ if (webService != null)
+ webService.stop();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlMySQLLT.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java?rev=1367952&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java Wed Aug 1 12:23:55 2012
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlPostgresqlLT extends BasePostgresql
+{
+
+ protected BigCrawlTester tester;
+ protected MockWebService webService = null;
+
+ public BigCrawlPostgresqlLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
+
+ // Setup and teardown the mock wiki service
+
+ @Before
+ public void createWebService()
+ throws Exception
+ {
+ webService = new MockWebService(10);
+ webService.start();
+ }
+
+ @After
+ public void shutdownWebService()
+ throws Exception
+ {
+ if (webService != null)
+ webService.stop();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlPostgresqlLT.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java?rev=1367952&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java Wed Aug 1 12:23:55 2012
@@ -0,0 +1,133 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import org.apache.manifoldcf.crawler.connectors.webcrawler.WebcrawlerConnector;
+import org.apache.manifoldcf.crawler.connectors.webcrawler.WebcrawlerConfig;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a 100000-document crawl */
+public class BigCrawlTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public BigCrawlTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("Web Connection");
+ conn.setDescription("Web Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.webcrawler.WebcrawlerConnector");
+ conn.setMaxConnections(100);
+ ConfigParams cp = conn.getConfigParams();
+
+ cp.setParameter(WebcrawlerConfig.PARAMETER_EMAIL,"someone@somewhere.com");
+ cp.setParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE,"none");
+
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("Web Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+ job.addHopCountFilter("link",new Long(4));
+ job.addHopCountFilter("redirect",new Long(2));
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+
+ // For 100000 documents, set up 10 seeds
+ SpecificationNode sn = new SpecificationNode(WebcrawlerConfig.NODE_SEEDS);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0 ; i < 10 ; i++)
+ {
+ sb.append("http://localhost:8191/web/gen.php?site="+i+"&level=0&item=0\n");
+ }
+ sn.setValue(sb.toString());
+ ds.addChild(ds.getChildCount(),sn);
+
+ sn = new SpecificationNode(WebcrawlerConfig.NODE_INCLUDES);
+ sn.setValue(".*\n");
+ ds.addChild(ds.getChildCount(),sn);
+
+ sn = new SpecificationNode(WebcrawlerConfig.NODE_INCLUDESINDEX);
+ sn.setValue(".*\n");
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ long startTime = System.currentTimeMillis();
+ jobManager.manualStart(job.getID());
+ instance.waitJobInactiveNative(jobManager,job.getID(),22000000L);
+ System.err.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // Four levels deep from 10 site seeds: Each site seed has 1 + 10 + 100 + 1000 + 10000 = 11111 documents, so 10 has 111110.
+ if (status.getDocumentsProcessed() != 111110)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 111110, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),18000000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java?rev=1367952&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java Wed Aug 1 12:23:55 2012
@@ -0,0 +1,161 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+
+import java.io.*;
+import java.util.*;
+
+/** Mock wiki service */
+public class MockWebService
+{
+ Server server;
+ WebServlet servlet;
+
+ public MockWebService(int docsPerLevel)
+ {
+ server = new Server(8191);
+ server.setThreadPool(new QueuedThreadPool(35));
+ servlet = new WebServlet(docsPerLevel);
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setContextPath("/web");
+ server.setHandler(context);
+ context.addServlet(new ServletHolder(servlet), "/gen.php");
+ }
+
+ public void start() throws Exception
+ {
+ server.start();
+ }
+
+ public void stop() throws Exception
+ {
+ server.stop();
+ }
+
+
+ public static class WebServlet extends HttpServlet
+ {
+ int docsPerLevel;
+
+ public WebServlet(int docsPerLevel)
+ {
+ this.docsPerLevel = docsPerLevel;
+ }
+
+ @Override
+ public void service(HttpServletRequest req, HttpServletResponse res)
+ throws IOException
+ {
+ String resourceName = null;
+
+ String site = req.getParameter("site"); // Site ID
+ if (site == null)
+ throw new IOException("Site ID parameter must be set");
+
+ String level = req.getParameter("level"); // Level #
+ if (site == null)
+ throw new IOException("Level number parameter must be set");
+
+ String item = req.getParameter("item"); // Item #
+ if (item == null)
+ throw new IOException("Item number parameter must be set");
+
+ int theLevel;
+ try
+ {
+ theLevel = Integer.parseInt(level);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new IOException("Level number must be a number: "+level);
+ }
+
+ int theItem;
+ try
+ {
+ theItem = Integer.parseInt(item);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new IOException("Item number must be a number: "+item);
+ }
+
+ // Formulate the response.
+ // First, calculate the number of docs on the current level
+ int maxDocsThisLevel = docsPerLevel ^ theLevel;
+ if (theItem >= maxDocsThisLevel)
+ // Not legal
+ throw new IOException("Level number too big: "+level);
+
+ // Generate the page
+ res.setStatus(HttpServletResponse.SC_OK);
+ res.setContentType("text/html; charset=utf-8");
+ res.getWriter().printf("<html>\n");
+ res.getWriter().printf(" <body>\n");
+
+ res.getWriter().printf("This is doc number "+theItem+" and level number "+theLevel+" in site "+site+"\n");
+
+ // Generate links to all parents
+ int parentLevel = theLevel;
+ int parentItem = theItem;
+ while (parentLevel > 0)
+ {
+ parentLevel--;
+ parentItem = parentItem/docsPerLevel;
+ generateLink(res,site,parentLevel,parentItem);
+ }
+
+ // Generate links to direct children
+ for (int i = 0; i < docsPerLevel; i++)
+ {
+ int docNumber = i + theItem * docsPerLevel;
+ generateLink(res,site,theLevel+1,docNumber);
+ }
+
+ // Generate some cross-links to other items' children
+ for (int i = 0; i < docsPerLevel; i++)
+ {
+ int docNumber = theItem + i * docsPerLevel;
+ generateLink(res,site,theLevel+1,docNumber);
+ }
+
+ res.getWriter().printf(" </body>\n");
+ res.getWriter().printf("</html>\n");
+ res.getWriter().flush();
+
+ }
+
+ protected void generateLink(HttpServletResponse res, String site, int level, int item)
+ throws IOException
+ {
+ res.getWriter().printf(" <a href=\"http://localhost:8191/web/gen.php?site="+site+"&level="+level+"&item="+item+"\"/>\n");
+ }
+
+ }
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java
------------------------------------------------------------------------------
svn:keywords = Id