You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2010/07/07 18:50:42 UTC
svn commit: r961432 - in /incubator/lcf/trunk/modules:
framework/tests/org/apache/lcf/crawler/tests/ testdata/ testdata/testdir/
tests/org/apache/lcf/tests/
Author: kwright
Date: Wed Jul 7 16:50:42 2010
New Revision: 961432
URL: http://svn.apache.org/viewvc?rev=961432&view=rev
Log:
Move the job cleanup in the testing infrastructure into a place where we will have a daemon running, otherwise the cleanup doesn't work. Also, add an actual job run to the sanity test.
Added:
incubator/lcf/trunk/modules/testdata/
incubator/lcf/trunk/modules/testdata/test1.txt
incubator/lcf/trunk/modules/testdata/test2.txt
incubator/lcf/trunk/modules/testdata/testdir/
incubator/lcf/trunk/modules/testdata/testdir/test3.txt
Modified:
incubator/lcf/trunk/modules/framework/tests/org/apache/lcf/crawler/tests/TestConnectorBase.java
incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/Sanity.java
incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/TestBase.java
Modified: incubator/lcf/trunk/modules/framework/tests/org/apache/lcf/crawler/tests/TestConnectorBase.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/tests/org/apache/lcf/crawler/tests/TestConnectorBase.java?rev=961432&r1=961431&r2=961432&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/tests/org/apache/lcf/crawler/tests/TestConnectorBase.java (original)
+++ incubator/lcf/trunk/modules/framework/tests/org/apache/lcf/crawler/tests/TestConnectorBase.java Wed Jul 7 16:50:42 2010
@@ -202,88 +202,13 @@ public class TestConnectorBase extends o
// First, tear down all jobs, connections, authority connections, and output connections.
try
{
- IJobManager jobManager = JobManagerFactory.make(tc);
IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(tc);
IAuthorityConnectionManager authConnMgr = AuthorityConnectionManagerFactory.make(tc);
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- // Get a list of the current active jobs
- IJobDescription[] jobs = jobManager.getAllJobs();
- int i = 0;
- while (i < jobs.length)
- {
- IJobDescription desc = jobs[i++];
- // Abort this job, if it is running
- try
- {
- jobManager.manualAbort(desc.getID());
- }
- catch (LCFException e)
- {
- // This generally means that the job was not running
- }
- }
- i = 0;
- while (i < jobs.length)
- {
- IJobDescription desc = jobs[i++];
- // Wait for this job to stop
- while (true)
- {
- JobStatus status = jobManager.getStatus(desc.getID());
- if (status != null)
- {
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- case JobStatus.JOBSTATUS_COMPLETED:
- case JobStatus.JOBSTATUS_ERROR:
- break;
- default:
- LCF.sleep(10000);
- continue;
- }
- }
- break;
- }
- }
-
- // Now, delete them all
- i = 0;
- while (i < jobs.length)
- {
- IJobDescription desc = jobs[i++];
- try
- {
- jobManager.deleteJob(desc.getID());
- }
- catch (LCFException e)
- {
- // This usually means that the job is already being deleted
- }
- }
-
- i = 0;
- while (i < jobs.length)
- {
- IJobDescription desc = jobs[i++];
- // Wait for this job to disappear
- while (true)
- {
- JobStatus status = jobManager.getStatus(desc.getID());
- if (status != null)
- {
- LCF.sleep(10000);
- continue;
- }
- break;
- }
- }
-
// Now, get a list of the repository connections
IRepositoryConnection[] connections = connMgr.getAllConnections();
- i = 0;
+ int i = 0;
while (i < connections.length)
{
connMgr.delete(connections[i++].getName());
Added: incubator/lcf/trunk/modules/testdata/test1.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/testdata/test1.txt?rev=961432&view=auto
==============================================================================
--- incubator/lcf/trunk/modules/testdata/test1.txt (added)
+++ incubator/lcf/trunk/modules/testdata/test1.txt Wed Jul 7 16:50:42 2010
@@ -0,0 +1 @@
+This is a test file
Added: incubator/lcf/trunk/modules/testdata/test2.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/testdata/test2.txt?rev=961432&view=auto
==============================================================================
--- incubator/lcf/trunk/modules/testdata/test2.txt (added)
+++ incubator/lcf/trunk/modules/testdata/test2.txt Wed Jul 7 16:50:42 2010
@@ -0,0 +1 @@
+This is another test file
Added: incubator/lcf/trunk/modules/testdata/testdir/test3.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/testdata/testdir/test3.txt?rev=961432&view=auto
==============================================================================
--- incubator/lcf/trunk/modules/testdata/testdir/test3.txt (added)
+++ incubator/lcf/trunk/modules/testdata/testdir/test3.txt Wed Jul 7 16:50:42 2010
@@ -0,0 +1 @@
+This is a third test file
Modified: incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/Sanity.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/Sanity.java?rev=961432&r1=961431&r2=961432&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/Sanity.java (original)
+++ incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/Sanity.java Wed Jul 7 16:50:42 2010
@@ -36,34 +36,135 @@ public class Sanity extends TestBase
public void sanityCheck()
throws Exception
{
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.lcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.lcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
+ try
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.lcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.lcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
- // MHL
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("../testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new LCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new LCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("exclude");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*.svn");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID());
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 5)
+ throw new LCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // May want to do an incremental crawl or two also.
+ // MHL
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ waitJobDeleted(jobManager,job.getID());
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
}
+ protected void waitJobInactive(IJobManager jobManager, Long jobID)
+ throws LCFException, InterruptedException
+ {
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ throw new LCFException("No such job: '"+jobID+"'");
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ throw new LCFException("Job was never started.");
+ case JobStatus.JOBSTATUS_COMPLETED:
+ break;
+ case JobStatus.JOBSTATUS_ERROR:
+ throw new LCFException("Job reports error status: "+status.getErrorText());
+ default:
+ LCF.sleep(10000L);
+ continue;
+ }
+ break;
+ }
+ }
+
+ protected void waitJobDeleted(IJobManager jobManager, Long jobID)
+ throws LCFException, InterruptedException
+ {
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ break;
+ LCF.sleep(10000L);
+ }
+ }
+
}
Modified: incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/TestBase.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/TestBase.java?rev=961432&r1=961431&r2=961432&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/TestBase.java (original)
+++ incubator/lcf/trunk/modules/tests/org/apache/lcf/tests/TestBase.java Wed Jul 7 16:50:42 2010
@@ -73,42 +73,123 @@ public class TestBase extends org.apache
public void cleanUp()
throws Exception
{
- // Shut down daemon
- Exception currentException = null;
-
- if (daemonThread != null)
+ initialize();
+ if (isInitialized())
{
+ Exception currentException = null;
IThreadContext tc = ThreadContextFactory.make();
- ILockManager lockManager = LockManagerFactory.make(tc);
- lockManager.setGlobalFlag(agentShutdownSignal);
-
- // Wait for daemon thread to exit.
- while (true)
+
+ // Delete all jobs (and wait for them to go away)
+ if (daemonThread != null)
{
- if (daemonThread.isAlive())
+ IJobManager jobManager = JobManagerFactory.make(tc);
+
+ // Get a list of the current active jobs
+ IJobDescription[] jobs = jobManager.getAllJobs();
+ int i = 0;
+ while (i < jobs.length)
{
- Thread.sleep(1000L);
- continue;
+ IJobDescription desc = jobs[i++];
+ // Abort this job, if it is running
+ try
+ {
+ jobManager.manualAbort(desc.getID());
+ }
+ catch (LCFException e)
+ {
+ // This generally means that the job was not running
+ }
+ }
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Wait for this job to stop
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(desc.getID());
+ if (status != null)
+ {
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ case JobStatus.JOBSTATUS_COMPLETED:
+ case JobStatus.JOBSTATUS_ERROR:
+ break;
+ default:
+ LCF.sleep(10000);
+ continue;
+ }
+ }
+ break;
+ }
}
- break;
- }
- Exception e = daemonThread.getDaemonException();
- if (e != null)
- currentException = e;
- }
- // Clean up everything else
- try
- {
- super.cleanUp();
- }
- catch (Exception e)
- {
- if (currentException == null)
- currentException = e;
+ // Now, delete them all
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ try
+ {
+ jobManager.deleteJob(desc.getID());
+ }
+ catch (LCFException e)
+ {
+ // This usually means that the job is already being deleted
+ }
+ }
+
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Wait for this job to disappear
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(desc.getID());
+ if (status != null)
+ {
+ LCF.sleep(10000);
+ continue;
+ }
+ break;
+ }
+ }
+
+ // Shut down daemon
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ lockManager.setGlobalFlag(agentShutdownSignal);
+
+ // Wait for daemon thread to exit.
+ while (true)
+ {
+ if (daemonThread.isAlive())
+ {
+ Thread.sleep(1000L);
+ continue;
+ }
+ break;
+ }
+
+ Exception e = daemonThread.getDaemonException();
+ if (e != null)
+ currentException = e;
+ }
+ // Clean up everything else
+ try
+ {
+ super.cleanUp();
+ }
+ catch (Exception e)
+ {
+ if (currentException == null)
+ currentException = e;
+ }
+ if (currentException != null)
+ throw currentException;
}
- if (currentException != null)
- throw currentException;
}
protected static class DaemonThread extends Thread