You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/10/13 04:07:39 UTC
svn commit: r1182635 - in /incubator/lcf/branches/CONNECTORS-256:
connectors/wiki/ tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/
Author: kwright
Date: Thu Oct 13 02:07:39 2011
New Revision: 1182635
URL: http://svn.apache.org/viewvc?rev=1182635&view=rev
Log:
More work on tests. Include test that does a basic crawl.
Added:
incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java (with props)
Modified:
incubator/lcf/branches/CONNECTORS-256/connectors/wiki/ (props changed)
incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java
incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
Propchange: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Thu Oct 13 02:07:39 2011
@@ -1,3 +1,4 @@
build
dist
lib
+test-output
Modified: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java?rev=1182635&r1=1182634&r2=1182635&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java (original)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java Thu Oct 13 02:07:39 2011
@@ -46,7 +46,8 @@ public class BaseDerby extends org.apach
protected DaemonThread daemonThread = null;
protected Server server = null;
-
+ protected MockWikiService wikiService = null;
+
protected String[] getConnectorNames()
{
return new String[]{"Wiki Connector"};
@@ -67,6 +68,24 @@ public class BaseDerby extends org.apach
return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
}
+ // Setup and teardown the mock wiki service
+
+ @Before
+ public void createWikiService()
+ throws Exception
+ {
+ wikiService = new MockWikiService(getClass());
+ wikiService.start();
+ }
+
+ @After
+ public void shutdownWikiService()
+ throws Exception
+ {
+ if (wikiService != null)
+ wikiService.stop();
+ }
+
// API support
// These methods allow communication with the ManifoldCF api webapp, via the locally-instantiated jetty
Modified: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java?rev=1182635&r1=1182634&r2=1182635&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java (original)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java Thu Oct 13 02:07:39 2011
@@ -56,9 +56,10 @@ public class MockWikiService
public void setResources(Map<String,String> checkResources,
Map<String,String> listResources,
Map<String,String> timestampQueryResources,
+ Map<String,String> urlQueryResources,
Map<String,String> docInfoQueryResources)
{
- servlet.setResources(checkResources,listResources,timestampQueryResources,docInfoQueryResources);
+ servlet.setResources(checkResources,listResources,timestampQueryResources,urlQueryResources,docInfoQueryResources);
}
protected static String sortStuff(String input)
@@ -82,6 +83,7 @@ public class MockWikiService
protected Map<String,String> checkResources = null;
protected Map<String,String> listResources = null;
protected Map<String,String> timestampQueryResources = null;
+ protected Map<String,String> urlQueryResources = null;
protected Map<String,String> docInfoQueryResources = null;
public WikiAPIServlet(Class theResourceClass)
@@ -92,11 +94,13 @@ public class MockWikiService
public void setResources(Map<String,String> checkResources,
Map<String,String> listResources,
Map<String,String> timestampQueryResources,
+ Map<String,String> urlQueryResources,
Map<String,String> docInfoQueryResources)
{
this.checkResources = checkResources;
this.listResources = listResources;
this.timestampQueryResources = timestampQueryResources;
+ this.urlQueryResources = urlQueryResources;
this.docInfoQueryResources = docInfoQueryResources;
}
@@ -121,32 +125,44 @@ public class MockWikiService
throw new IOException("Action parameter incorrect: "+action);
String prop = req.getParameter("prop");
String pageIds = req.getParameter("pageids");
- String rvprop = req.getParameter("rvprop");
- String inprop = req.getParameter("inprop");
- if (rvprop != null && rvprop.equals("timestamp"))
+ if (prop != null && prop.equals("revisions"))
{
- // Version query
- if (prop == null || !prop.equals("revisions"))
- throw new IOException("prop parameter incorrect for version query: "+prop);
- if (pageIds == null)
- throw new IOException("missing pageids parameter, required for version query");
+ String rvprop = req.getParameter("rvprop");
+ if (rvprop != null && rvprop.equals("timestamp"))
+ {
+ // Version query
+ if (pageIds == null)
+ throw new IOException("missing pageids parameter, required for version query");
+ // Sort the pageIds
+ pageIds = sortStuff(pageIds);
+ resourceName = timestampQueryResources.get(pageIds);
+ }
+ else if (rvprop != null && rvprop.equals("user|comment|content"))
+ {
+ // Doc info query
+ if (pageIds == null)
+ throw new IOException("missing pageids parameter, required for docinfo query");
+ if (pageIds.indexOf("|") != -1)
+ throw new IOException("cannot do more than one docinfo request at once");
+ resourceName = docInfoQueryResources.get(pageIds);
+ }
+ else
+ throw new IOException("rvprop parameter missing or incorrect: "+rvprop);
+ }
+ else if (prop != null && prop.equals("info"))
+ {
+ String inprop = req.getParameter("inprop");
+ if (inprop == null || !inprop.equals("url"))
+ throw new IOException("inprop parameter missing or incorrect: "+inprop);
+ // url query
+ if (pageIds == null)
+ throw new IOException("missing pageids parameter, required for url query");
// Sort the pageIds
pageIds = sortStuff(pageIds);
- resourceName = timestampQueryResources.get(pageIds);
- }
- else if (rvprop != null && rvprop.equals("user|comment|content"))
- {
- // Doc info query
- if (prop == null || !prop.equals("revisions"))
- throw new IOException("prop parameter incorrect for docinfo query: "+prop);
- if (pageIds == null)
- throw new IOException("missing pageids parameter, required for docinfo query");
- if (pageIds.indexOf("|") != -1)
- throw new IOException("cannot do more than one docinfo request at once");
- resourceName = docInfoQueryResources.get(pageIds);
+ resourceName = urlQueryResources.get(pageIds);
}
else
- throw new IOException("rvprop parameter missing or incorrect: "+rvprop);
+ throw new IOException("Invalid value for prop parameter: "+prop);
}
else if (list != null)
{
Added: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java?rev=1182635&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java (added)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java Thu Oct 13 02:07:39 2011
@@ -0,0 +1,230 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.wiki_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import org.apache.manifoldcf.crawler.connectors.wiki.WikiConfig;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class SanityDerbyTest extends BaseDerby
+{
+ protected static Map<String,String> initialCheckResources;
+ protected static Map<String,String> initialListResources;
+ protected static Map<String,String> initialTimestampQueryResources;
+ protected static Map<String,String> initialURLQueryResources;
+ protected static Map<String,String> initialDocInfoQueryResources;
+
+ static
+ {
+ initialCheckResources.put("","list_one.xml");
+
+ initialListResources.put("","list_full.xml");
+ initialListResources.put("Kre Mbaye","list_full_last.xml");
+
+ initialTimestampQueryResources.put("14773725|19219017|19319577|19839654|30955295","get_timestamps.xml");
+
+ initialURLQueryResources.put("14773725|19219017|19319577|19839654|30955295","get_urls.xml");
+
+ initialDocInfoQueryResources.put("14773725","14773725.xml");
+ initialDocInfoQueryResources.put("19219017","19219017.xml");
+ initialDocInfoQueryResources.put("19319577","19319577.xml");
+ initialDocInfoQueryResources.put("19839654","19839654.xml");
+ initialDocInfoQueryResources.put("30955295","30955295.xml");
+
+ }
+
+ @Test
+ public void sanityCheck()
+ throws Exception
+ {
+ try
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("Wiki Connection");
+ conn.setDescription("Wiki Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.wiki.WikiConnector");
+ conn.setMaxConnections(10);
+ ConfigParams cp = conn.getConfigParams();
+ cp.setParameter(WikiConfig.PARAM_PROTOCOL,"http");
+ cp.setParameter(WikiConfig.PARAM_SERVER,"localhost");
+ cp.setParameter(WikiConfig.PARAM_PORT,"8089");
+ cp.setParameter(WikiConfig.PARAM_PATH,"/w");
+
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("Wiki Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+
+ // Now, set up the document specification.
+ // Right now we don't need any...
+ DocumentSpecification ds = job.getSpecification();
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Initialize the mock service
+ wikiService.setResources(initialCheckResources,
+ initialListResources,
+ initialTimestampQueryResources,
+ initialURLQueryResources,
+ initialDocInfoQueryResources);
+
+ // Now, start the job, and wait until it completes.
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID(),120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 5)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ /*
+ // Add a file and recrawl
+ createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+ // Now, start the job, and wait until it completes.
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID(),120000L);
+
+ status = jobManager.getStatus(job.getID());
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Change a file, and recrawl
+ changeFile(new File("testdata/test1.txt"),"Modified contents");
+
+ // Now, start the job, and wait until it completes.
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID(),120000L);
+
+ status = jobManager.getStatus(job.getID());
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+ // We also need to make sure the new document was indexed. Have to think about how to do this though.
+ // MHL
+
+ // Delete a file, and recrawl
+ removeFile(new File("testdata/test2.txt"));
+
+ // Now, start the job, and wait until it completes.
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID(),120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 5)
+ throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
+ */
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ waitJobDeleted(jobManager,job.getID(),120000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+ throws ManifoldCFException, InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ throw new ManifoldCFException("No such job: '"+jobID+"'");
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ throw new ManifoldCFException("Job was never started.");
+ case JobStatus.JOBSTATUS_COMPLETED:
+ break;
+ case JobStatus.JOBSTATUS_ERROR:
+ throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+ default:
+ ManifoldCF.sleep(1000L);
+ continue;
+ }
+ return;
+ }
+ throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+ protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+ throws ManifoldCFException, InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ return;
+ ManifoldCF.sleep(1000L);
+ }
+ throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+
+}
Propchange: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
------------------------------------------------------------------------------
svn:keywords = Id