You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/10/13 08:33:28 UTC
svn commit: r1182680 - in /incubator/lcf/branches/CONNECTORS-256: ./
connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/
tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/
tests/wiki/src/test/resources/org/apa...
Author: kwright
Date: Thu Oct 13 06:33:28 2011
New Revision: 1182680
URL: http://svn.apache.org/viewvc?rev=1182680&view=rev
Log:
Everything works, except the random possibility of pages getting batched or not batched, which makes it a test issue. Gotta think about this one.
Added:
incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/resources/org/apache/manifoldcf/wiki_tests/get_urls.xml (with props)
Modified:
incubator/lcf/branches/CONNECTORS-256/build.xml
incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
Modified: incubator/lcf/branches/CONNECTORS-256/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/build.xml?rev=1182680&r1=1182679&r2=1182680&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/build.xml (original)
+++ incubator/lcf/branches/CONNECTORS-256/build.xml Thu Oct 13 06:33:28 2011
@@ -2280,7 +2280,9 @@
</classpath>
<formatter type="brief" usefile="false"/>
+ <test name="org.apache.manifoldcf.wiki_tests.SanityDerbyTest" todir="test-output/wiki"/>
<!-- MHL -->
+
</junit>
</target>
Modified: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1182680&r1=1182679&r2=1182680&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java (original)
+++ incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java Thu Oct 13 06:33:28 2011
@@ -184,6 +184,7 @@ public class WikiConnector extends org.a
/** Get the maximum number of documents to amalgamate together into one batch, for this connector.
*@return the maximum number. 0 indicates "unlimited".
*/
+ @Override
public int getMaxDocumentRequest()
{
return 20;
Modified: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java?rev=1182680&r1=1182679&r2=1182680&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java (original)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java Thu Oct 13 06:33:28 2011
@@ -69,7 +69,7 @@ public class MockWikiService
StringBuilder sb = new StringBuilder();
for (int i = 0 ; i < ids.length ; i++)
{
- if (i == 0)
+ if (i > 0)
sb.append("|");
sb.append(ids[i]);
}
@@ -113,19 +113,19 @@ public class MockWikiService
String format = req.getParameter("format");
if (!format.equals("xml"))
throw new IOException("Format parameter incorrect: "+format);
- String list = req.getParameter("list");
String action = req.getParameter("action");
- if (action == null && list == null)
- throw new IOException("Must have either action or list");
- if (action != null && list != null)
- throw new IOException("Cannot have both action and list");
- if (action != null)
+ if (action == null || !action.equals("query"))
+ throw new IOException("Action parameter incorrect: "+action);
+ String list = req.getParameter("list");
+ String prop = req.getParameter("prop");
+ if (prop == null && list == null)
+ throw new IOException("Must have either prop or list");
+ if (prop != null && list != null)
+ throw new IOException("Cannot have both prop and list");
+ if (prop != null)
{
- if (!action.equals("query"))
- throw new IOException("Action parameter incorrect: "+action);
- String prop = req.getParameter("prop");
String pageIds = req.getParameter("pageids");
- if (prop != null && prop.equals("revisions"))
+ if (prop.equals("revisions"))
{
String rvprop = req.getParameter("rvprop");
if (rvprop != null && rvprop.equals("timestamp"))
@@ -136,6 +136,8 @@ public class MockWikiService
// Sort the pageIds
pageIds = sortStuff(pageIds);
resourceName = timestampQueryResources.get(pageIds);
+ if (resourceName == null)
+ throw new IOException("Could not find a matching resource for the timestamp parameters; pageids = '"+pageIds+"'");
}
else if (rvprop != null && rvprop.equals("user|comment|content"))
{
@@ -145,21 +147,25 @@ public class MockWikiService
if (pageIds.indexOf("|") != -1)
throw new IOException("cannot do more than one docinfo request at once");
resourceName = docInfoQueryResources.get(pageIds);
+ if (resourceName == null)
+ throw new IOException("Could not find a matching resource for the user|comment|content parameters; pageids = '"+pageIds+"'");
}
else
throw new IOException("rvprop parameter missing or incorrect: "+rvprop);
}
- else if (prop != null && prop.equals("info"))
+ else if (prop.equals("info"))
{
String inprop = req.getParameter("inprop");
if (inprop == null || !inprop.equals("url"))
throw new IOException("inprop parameter missing or incorrect: "+inprop);
// url query
- if (pageIds == null)
- throw new IOException("missing pageids parameter, required for url query");
+ if (pageIds == null)
+ throw new IOException("missing pageids parameter, required for url query");
// Sort the pageIds
pageIds = sortStuff(pageIds);
resourceName = urlQueryResources.get(pageIds);
+ if (resourceName == null)
+ throw new IOException("Could not find a matching resource for the info url parameters; pageids = '"+pageIds+"'");
}
else
throw new IOException("Invalid value for prop parameter: "+prop);
@@ -179,6 +185,9 @@ public class MockWikiService
resourceName = listResources.get(apfrom);
else
throw new IOException("aplimit parameter incorrect: "+aplimit);
+ if (resourceName == null)
+ throw new IOException("Could not find a matching resource for the list parameters; apfrom = '"+apfrom+"'");
+
}
// Select the resource
Modified: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java?rev=1182680&r1=1182679&r2=1182680&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java (original)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java Thu Oct 13 06:33:28 2011
@@ -40,15 +40,20 @@ public class SanityDerbyTest extends Bas
static
{
+ initialCheckResources = new HashMap<String,String>();
initialCheckResources.put("","list_one.xml");
+ initialListResources = new HashMap<String,String>();
initialListResources.put("","list_full.xml");
initialListResources.put("Kre Mbaye","list_full_last.xml");
+ initialTimestampQueryResources = new HashMap<String,String>();
initialTimestampQueryResources.put("14773725|19219017|19319577|19839654|30955295","get_timestamps.xml");
+ initialURLQueryResources = new HashMap<String,String>();
initialURLQueryResources.put("14773725|19219017|19319577|19839654|30955295","get_urls.xml");
+ initialDocInfoQueryResources = new HashMap<String,String>();
initialDocInfoQueryResources.put("14773725","14773725.xml");
initialDocInfoQueryResources.put("19219017","19219017.xml");
initialDocInfoQueryResources.put("19319577","19319577.xml");
Added: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/resources/org/apache/manifoldcf/wiki_tests/get_urls.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/resources/org/apache/manifoldcf/wiki_tests/get_urls.xml?rev=1182680&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/resources/org/apache/manifoldcf/wiki_tests/get_urls.xml (added)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/resources/org/apache/manifoldcf/wiki_tests/get_urls.xml Thu Oct 13 06:33:28 2011
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<api><query><pages><page pageid="14773725" ns="0" title="Kre8tiveworkz" touched="2011-10-03T03:51:34Z" lastrevid="376463852" counter="" length="6357" fullurl="http://en.wikipedia.org/wiki/Kre8tiveworkz" editurl="http://en.wikipedia.org/w/index.php?title=Kre8tiveworkz&action=edit"/><page pageid="19219017" ns="0" title="Kre M'Baye" touched="2011-10-03T03:30:56Z" lastrevid="431554656" counter="" length="7749" fullurl="http://en.wikipedia.org/wiki/Kre_M%27Baye" editurl="http://en.wikipedia.org/w/index.php?title=Kre_M%27Baye&action=edit"/><page pageid="19319577" ns="0" title="Kre Mbaye" touched="2011-05-29T22:07:53Z" lastrevid="238432075" counter="" length="24" redirect="" new="" fullurl="http://en.wikipedia.org/wiki/Kre_Mbaye" editurl="http://en.wikipedia.org/w/index.php?title=Kre_Mbaye&action=edit"/><page pageid="19839654" ns="0" title="Kre'fey" touched="2011-09-29T00:50:43Z" lastrevid="246209449" counter="" length="32" redirect="" new="" fullurl="http://en.wikipedi
a.org/wiki/Kre%27fey" editurl="http://en.wikipedia.org/w/index.php?title=Kre%27fey&action=edit"/><page pageid="30955295" ns="0" title="Kre-O" touched="2011-10-12T07:13:26Z" lastrevid="448739088" counter="" length="4995" fullurl="http://en.wikipedia.org/wiki/Kre-O" editurl="http://en.wikipedia.org/w/index.php?title=Kre-O&action=edit"/></pages></query></api>
\ No newline at end of file
Propchange: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/resources/org/apache/manifoldcf/wiki_tests/get_urls.xml
------------------------------------------------------------------------------
svn:binary =