You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/08/08 17:05:45 UTC
svn commit: r1370798 - in
/manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests:
BigCrawlTester.java MockWebService.java
Author: kwright
Date: Wed Aug 8 15:05:43 2012
New Revision: 1370798
URL: http://svn.apache.org/viewvc?rev=1370798&view=rev
Log:
Reduce test to the minimal size to cause the problem
Modified:
manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java
Modified: manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java?rev=1370798&r1=1370797&r2=1370798&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java (original)
+++ manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/BigCrawlTester.java Wed Aug 8 15:05:43 2012
@@ -80,7 +80,7 @@ public class BigCrawlTester
job.setType(job.TYPE_SPECIFIED);
job.setStartMethod(job.START_DISABLE);
job.setHopcountMode(job.HOPCOUNT_ACCURATE);
- job.addHopCountFilter("link",new Long(3));
+ job.addHopCountFilter("link",new Long(2));
job.addHopCountFilter("redirect",new Long(2));
// Now, set up the document specification.
@@ -120,9 +120,17 @@ public class BigCrawlTester
// Check to be sure we actually processed the right number of documents.
JobStatus status = jobManager.getStatus(job.getID());
// Four levels deep from 10 site seeds: Each site seed has 1 + 10 + 100 + 1000 = 1111 documents, so 10 has 11110.
- if (status.getDocumentsProcessed() != 11110)
- throw new ManifoldCFException("Wrong number of documents processed - expected 11110, saw "+new Long(status.getDocumentsProcessed()).toString());
-
+ if (status.getDocumentsProcessed() != 1110)
+ {
+ System.err.println("Sleeping for database inspection");
+ while (true)
+ {
+ if (1 < 0)
+ Thread.sleep(10000L);
+ }
+ throw new ManifoldCFException("Wrong number of documents processed - expected 1110, saw "+new Long(status.getDocumentsProcessed()).toString());
+ }
+
// Now, delete the job.
jobManager.deleteJob(job.getID());
instance.waitJobDeletedNative(jobManager,job.getID(),18000000L);
Modified: manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java?rev=1370798&r1=1370797&r2=1370798&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java (original)
+++ manifoldcf/branches/CONNECTORS-501/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockWebService.java Wed Aug 8 15:05:43 2012
@@ -136,11 +136,16 @@ public class MockWebService
generateLink(res,site,parentLevel,parentItem);
}
- // Generate links to direct children
- for (int i = 0; i < docsPerLevel; i++)
+ // Temporary: Prevent links to children deeper than a certain level; this is to help
+ // the debug process
+ if (theLevel < 2)
{
- int docNumber = i + theItem * docsPerLevel;
- generateLink(res,site,theLevel+1,docNumber);
+ // Generate links to direct children
+ for (int i = 0; i < docsPerLevel; i++)
+ {
+ int docNumber = i + theItem * docsPerLevel;
+ generateLink(res,site,theLevel+1,docNumber);
+ }
}
// Generate some limited cross-links to other items at this level