You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/12/31 18:23:37 UTC
svn commit: r1226141 [3/5] - in /incubator/lcf/trunk: ./
connectors/alfresco/ connectors/alfresco/alfresco-ws-client/
connectors/cmis/ connectors/documentum/dfc/ connectors/filenet/filenet-api/
connectors/filesystem/ connectors/jcifs/jcifs/ connectors/...
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java Sat Dec 31 17:23:35 2011
@@ -18,11 +18,6 @@
*/
package org.apache.manifoldcf.filesystem_tests;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-
import java.io.*;
import java.util.*;
import org.junit.*;
@@ -30,211 +25,31 @@ import org.junit.*;
/** This is a very basic sanity check */
public class BigCrawlMySQLLT extends BaseMySQL
{
+ protected BigCrawlTester tester;
+
+ public BigCrawlMySQLLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- // Create the test data files.
- String baseFileName = "testdata/";
- int i0 = 0;
- while (i0 < 10)
- {
- String fileName0 = baseFileName + "/dir-" + i0;
- createDirectory(new File(fileName0));
- int i1 = 0;
- while (i1 < 10)
- {
- String fileName1 = fileName0 + "/dir-" + i1;
- createDirectory(new File(fileName1));
- int i2 = 0;
- while (i2 < 10)
- {
- String fileName2 = fileName1 + "/dir-" + i2;
- createDirectory(new File(fileName2));
- int i3 = 0;
- while (i3 < 10)
- {
- String fileName3 = fileName2 + "/dir-" + i3;
- createDirectory(new File(fileName3));
- int i4 = 0;
- while (i4 < 10)
- {
- String fileName4 = fileName3 + "/file-"+i4;
- createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4);
- i4++;
- }
- i3++;
- }
- i2++;
- }
- i1++;
- }
- i0++;
- }
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
- System.out.println("Done generating files");
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- System.out.println("Removing generated files");
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void bigCrawl()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Now, start the job, and wait until it completes.
- long startTime = System.currentTimeMillis();
- jobManager.manualStart(job.getID());
- waitJobInactive(jobManager,job.getID(),18000000L);
- System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
-
- // Check to be sure we actually processed the right number of documents.
- JobStatus status = jobManager.getStatus(job.getID());
- // The test data area has 3 documents and one directory, and we have to count the root directory too.
- if (status.getDocumentsProcessed() != 111111)
- throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString());
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(),18000000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
- }
-
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ tester.executeTest();
}
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
-
}
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java Sat Dec 31 17:23:35 2011
@@ -18,11 +18,6 @@
*/
package org.apache.manifoldcf.filesystem_tests;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-
import java.io.*;
import java.util.*;
import org.junit.*;
@@ -30,211 +25,31 @@ import org.junit.*;
/** This is a very basic sanity check */
public class BigCrawlPostgresqlLT extends BasePostgresql
{
+ protected BigCrawlTester tester;
+
+ public BigCrawlPostgresqlLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- // Create the test data files.
- String baseFileName = "testdata/";
- int i0 = 0;
- while (i0 < 10)
- {
- String fileName0 = baseFileName + "/dir-" + i0;
- createDirectory(new File(fileName0));
- int i1 = 0;
- while (i1 < 10)
- {
- String fileName1 = fileName0 + "/dir-" + i1;
- createDirectory(new File(fileName1));
- int i2 = 0;
- while (i2 < 10)
- {
- String fileName2 = fileName1 + "/dir-" + i2;
- createDirectory(new File(fileName2));
- int i3 = 0;
- while (i3 < 10)
- {
- String fileName3 = fileName2 + "/dir-" + i3;
- createDirectory(new File(fileName3));
- int i4 = 0;
- while (i4 < 10)
- {
- String fileName4 = fileName3 + "/file-"+i4;
- createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4);
- i4++;
- }
- i3++;
- }
- i2++;
- }
- i1++;
- }
- i0++;
- }
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
- System.out.println("Done generating files");
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- System.out.println("Removing generated files");
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void bigCrawl()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Now, start the job, and wait until it completes.
- long startTime = System.currentTimeMillis();
- jobManager.manualStart(job.getID());
- waitJobInactive(jobManager,job.getID(),18000000L);
- System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
-
- // Check to be sure we actually processed the right number of documents.
- JobStatus status = jobManager.getStatus(job.getID());
- // The test data area has 3 documents and one directory, and we have to count the root directory too.
- if (status.getDocumentsProcessed() != 111111)
- throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString());
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(),18000000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
- }
-
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ tester.executeTest();
}
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
-
}
Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java?rev=1226141&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java (added)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java Sat Dec 31 17:23:35 2011
@@ -0,0 +1,176 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a 100000 document crawl */
+public class BigCrawlTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public BigCrawlTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ // Create the test data files.
+ String baseFileName = "testdata/";
+ int i0 = 0;
+ while (i0 < 10)
+ {
+ String fileName0 = baseFileName + "/dir-" + i0;
+ FileHelper.createDirectory(new File(fileName0));
+ int i1 = 0;
+ while (i1 < 10)
+ {
+ String fileName1 = fileName0 + "/dir-" + i1;
+ FileHelper.createDirectory(new File(fileName1));
+ int i2 = 0;
+ while (i2 < 10)
+ {
+ String fileName2 = fileName1 + "/dir-" + i2;
+ FileHelper.createDirectory(new File(fileName2));
+ int i3 = 0;
+ while (i3 < 10)
+ {
+ String fileName3 = fileName2 + "/dir-" + i3;
+ FileHelper.createDirectory(new File(fileName3));
+ int i4 = 0;
+ while (i4 < 10)
+ {
+ String fileName4 = fileName3 + "/file-"+i4;
+ FileHelper.createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4);
+ i4++;
+ }
+ i3++;
+ }
+ i2++;
+ }
+ i1++;
+ }
+ i0++;
+ }
+ System.err.println("Done generating files");
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ System.err.println("Removing generated files");
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ long startTime = System.currentTimeMillis();
+ jobManager.manualStart(job.getID());
+ instance.waitJobInactiveNative(jobManager,job.getID(),18000000L);
+ System.err.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 111111)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),18000000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java Sat Dec 31 17:23:35 2011
@@ -18,11 +18,6 @@
*/
package org.apache.manifoldcf.filesystem_tests;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-
import java.io.*;
import java.util.*;
import org.junit.*;
@@ -30,230 +25,32 @@ import org.junit.*;
/** This is a test which checks to see if document expiration works properly. */
public class ExpirationHSQLDBIT extends BaseHSQLDB
{
+ protected ExpirationTester tester;
+
+ public ExpirationHSQLDBIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void expirationCheck()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_ACCURATE);
- // 60-second expiration for all documents..
- job.setExpiration(new Long(60000L));
- // Infinite rescan interval.
- job.setInterval(null);
- // Continuous job.
- job.setType(IJobDescription.TYPE_CONTINUOUS);
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Create the test data files.
- createFile(new File("testdata/test1.txt"),"This is a test file");
- createFile(new File("testdata/test2.txt"),"This is another test file");
- createDirectory(new File("testdata/testdir"));
- createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
- createDirectory(new File("testdata/testdir/seconddir"));
- createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
-
- // Now, start the job, and wait until it is running.
- jobManager.manualStart(job.getID());
- waitJobRunning(jobManager,job.getID(),30000L);
-
- JobStatus status;
-
- // Now we wait, and we should see 7 documents eventually.
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + 120000L)
- {
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() == 7)
- break;
-
- ManifoldCF.sleep(1000L);
- }
-
- // At this point there should be 7 documents.
- // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time.
- // So all we need to do is confirm that the job stops within 2 minutes.
- waitJobInactive(jobManager,job.getID(),180000L);
-
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() != 0)
- throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
-
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(),120000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
- }
-
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ tester.executeTest();
}
- protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- throw new ManifoldCFException("Job ended on its own!");
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- case JobStatus.JOBSTATUS_RUNNING:
- break;
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
-
}
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java Sat Dec 31 17:23:35 2011
@@ -18,11 +18,6 @@
*/
package org.apache.manifoldcf.filesystem_tests;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-
import java.io.*;
import java.util.*;
import org.junit.*;
@@ -30,230 +25,31 @@ import org.junit.*;
/** This is a test which checks to see if document expiration works properly. */
public class ExpirationIT extends BaseDerby
{
+ protected ExpirationTester tester;
+
+ public ExpirationIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void expirationCheck()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_ACCURATE);
- // 60-second expiration for all documents..
- job.setExpiration(new Long(60000L));
- // Infinite rescan interval.
- job.setInterval(null);
- // Continuous job.
- job.setType(IJobDescription.TYPE_CONTINUOUS);
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Create the test data files.
- createFile(new File("testdata/test1.txt"),"This is a test file");
- createFile(new File("testdata/test2.txt"),"This is another test file");
- createDirectory(new File("testdata/testdir"));
- createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
- createDirectory(new File("testdata/testdir/seconddir"));
- createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
-
- // Now, start the job, and wait until it is running.
- jobManager.manualStart(job.getID());
- waitJobRunning(jobManager,job.getID(),30000L);
-
- JobStatus status;
-
- // Now we wait, and we should see 7 documents eventually.
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + 120000L)
- {
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() == 7)
- break;
-
- ManifoldCF.sleep(1000L);
- }
-
- // At this point there should be 7 documents.
- // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time.
- // So all we need to do is confirm that the job stops within 2 minutes.
- waitJobInactive(jobManager,job.getID(),180000L);
-
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() != 0)
- throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
-
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(),120000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.executeTest();
}
-
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- throw new ManifoldCFException("Job ended on its own!");
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- case JobStatus.JOBSTATUS_RUNNING:
- break;
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
-
}
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java Sat Dec 31 17:23:35 2011
@@ -30,230 +30,31 @@ import org.junit.*;
/** This is a test which checks to see if document expiration works properly. */
public class ExpirationMySQLIT extends BaseMySQL
{
+ protected ExpirationTester tester;
+
+ public ExpirationMySQLIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void expirationCheck()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_ACCURATE);
- // 60-second expiration for all documents..
- job.setExpiration(new Long(60000L));
- // Infinite rescan interval.
- job.setInterval(null);
- // Continuous job.
- job.setType(IJobDescription.TYPE_CONTINUOUS);
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Create the test data files.
- createFile(new File("testdata/test1.txt"),"This is a test file");
- createFile(new File("testdata/test2.txt"),"This is another test file");
- createDirectory(new File("testdata/testdir"));
- createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
- createDirectory(new File("testdata/testdir/seconddir"));
- createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
-
- // Now, start the job, and wait until it is running.
- jobManager.manualStart(job.getID());
- waitJobRunning(jobManager,job.getID(),30000L);
-
- JobStatus status;
-
- // Now we wait, and we should see 7 documents eventually.
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + 120000L)
- {
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() == 7)
- break;
-
- ManifoldCF.sleep(1000L);
- }
-
- // At this point there should be 7 documents.
- // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time.
- // So all we need to do is confirm that the job stops within 2 minutes.
- waitJobInactive(jobManager,job.getID(),180000L);
-
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() != 0)
- throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
-
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(),120000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
- }
-
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- throw new ManifoldCFException("Job ended on its own!");
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- case JobStatus.JOBSTATUS_RUNNING:
- break;
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ tester.executeTest();
}
-
-
}
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java Sat Dec 31 17:23:35 2011
@@ -18,11 +18,6 @@
*/
package org.apache.manifoldcf.filesystem_tests;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-
import java.io.*;
import java.util.*;
import org.junit.*;
@@ -30,230 +25,31 @@ import org.junit.*;
/** This is a test which checks to see if document expiration works properly. */
public class ExpirationPostgresqlIT extends BasePostgresql
{
+ protected ExpirationTester tester;
+
+ public ExpirationPostgresqlIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void expirationCheck()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_ACCURATE);
- // 60-second expiration for all documents..
- job.setExpiration(new Long(60000L));
- // Infinite rescan interval.
- job.setInterval(null);
- // Continuous job.
- job.setType(IJobDescription.TYPE_CONTINUOUS);
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Create the test data files.
- createFile(new File("testdata/test1.txt"),"This is a test file");
- createFile(new File("testdata/test2.txt"),"This is another test file");
- createDirectory(new File("testdata/testdir"));
- createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
- createDirectory(new File("testdata/testdir/seconddir"));
- createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
-
- // Now, start the job, and wait until it is running.
- jobManager.manualStart(job.getID());
- waitJobRunning(jobManager,job.getID(),30000L);
-
- JobStatus status;
-
- // Now we wait, and we should see 7 documents eventually.
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + 120000L)
- {
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() == 7)
- break;
-
- ManifoldCF.sleep(1000L);
- }
-
- // At this point there should be 7 documents.
- // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time.
- // So all we need to do is confirm that the job stops within 2 minutes.
- waitJobInactive(jobManager,job.getID(),180000L);
-
- status = jobManager.getStatus(job.getID());
- if (status.getDocumentsProcessed() != 0)
- throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
-
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(),120000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.executeTest();
}
-
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- throw new ManifoldCFException("Job ended on its own!");
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- case JobStatus.JOBSTATUS_RUNNING:
- break;
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
-
}
Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java?rev=1226141&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java (added)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java Sat Dec 31 17:23:35 2011
@@ -0,0 +1,166 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public ExpirationTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+ // 60-second expiration for all documents..
+ job.setExpiration(new Long(60000L));
+ // Infinite rescan interval.
+ job.setInterval(null);
+ // Continuous job.
+ job.setType(IJobDescription.TYPE_CONTINUOUS);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Create the test data files.
+ FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file");
+ FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file");
+ FileHelper.createDirectory(new File("testdata/testdir"));
+ FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+ FileHelper.createDirectory(new File("testdata/testdir/seconddir"));
+ FileHelper.createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
+
+ // Now, start the job, and wait until it is running.
+ jobManager.manualStart(job.getID());
+ instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+
+ JobStatus status;
+
+ // Now we wait, and we should see 7 documents eventually.
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + 120000L)
+ {
+ status = jobManager.getStatus(job.getID());
+ if (status.getDocumentsProcessed() == 7)
+ break;
+
+ ManifoldCF.sleep(1000L);
+ }
+
+ // At this point there should be 7 documents.
+ // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time.
+ // So all we need to do is confirm that the job stops within 2 minutes.
+ instance.waitJobInactiveNative(jobManager,job.getID(),180000L);
+
+ status = jobManager.getStatus(job.getID());
+ if (status.getDocumentsProcessed() != 0)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),120000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java?rev=1226141&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java (added)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java Sat Dec 31 17:23:35 2011
@@ -0,0 +1,90 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+
+/** Helper class for local file manipulation */
+public class FileHelper
+{
+
+ protected static void createDirectory(File f)
+ throws Exception
+ {
+ if (f.mkdirs() == false)
+ throw new Exception("Failed to create directory "+f.toString());
+ }
+
+ protected static void removeDirectory(File f)
+ throws Exception
+ {
+ File[] files = f.listFiles();
+ if (files != null)
+ {
+ int i = 0;
+ while (i < files.length)
+ {
+ File subfile = files[i++];
+ if (subfile.isDirectory())
+ removeDirectory(subfile);
+ else
+ subfile.delete();
+ }
+ }
+ f.delete();
+ }
+
+ protected static void createFile(File f, String contents)
+ throws Exception
+ {
+ OutputStream os = new FileOutputStream(f);
+ try
+ {
+ Writer w = new OutputStreamWriter(os,"utf-8");
+ try
+ {
+ w.write(contents);
+ }
+ finally
+ {
+ w.flush();
+ }
+ }
+ finally
+ {
+ os.close();
+ }
+ }
+
+ protected static void removeFile(File f)
+ throws Exception
+ {
+ if (f.delete() == false)
+ throw new Exception("Failed to delete file "+f.toString());
+ }
+
+ protected static void changeFile(File f, String newContents)
+ throws Exception
+ {
+ removeFile(f);
+ createFile(f,newContents);
+ }
+
+}
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java (original)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java Sat Dec 31 17:23:35 2011
@@ -18,11 +18,6 @@
*/
package org.apache.manifoldcf.filesystem_tests;
-import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-
import java.io.*;
import java.util.*;
import org.junit.*;
@@ -30,180 +25,32 @@ import org.junit.*;
/** This is a test which checks to be sure hopcount functionality is working properly. */
public class HopcountHSQLDBIT extends BaseHSQLDB
{
+ protected HopcountTester tester;
+
+ public HopcountHSQLDBIT()
+ {
+ tester = new HopcountTester(mcfInstance);
+ }
@Before
- public void createTestArea()
+ public void setupTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- createDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.setupTestArea();
}
@After
- public void removeTestArea()
+ public void teardownTester()
throws Exception
{
- try
- {
- File f = new File("testdata");
- removeDirectory(f);
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.teardownTestArea();
}
@Test
public void hopcountCheck()
throws Exception
{
- try
- {
- // Hey, we were able to install the file system connector etc.
- // Now, create a local test job and run it.
- IThreadContext tc = ThreadContextFactory.make();
-
- // Create a basic file system connection, and save it.
- IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
- IRepositoryConnection conn = mgr.create();
- conn.setName("File Connection");
- conn.setDescription("File Connection");
- conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
- conn.setMaxConnections(100);
- // Now, save
- mgr.save(conn);
-
- // Create a basic null output connection, and save it.
- IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
- IOutputConnection outputConn = outputMgr.create();
- outputConn.setName("Null Connection");
- outputConn.setDescription("Null Connection");
- outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
- outputConn.setMaxConnections(100);
- // Now, save
- outputMgr.save(outputConn);
-
- // Create a job.
- IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription job = jobManager.createJob();
- job.setDescription("Test Job");
- job.setConnectionName("File Connection");
- job.setOutputConnectionName("Null Connection");
- job.setType(job.TYPE_SPECIFIED);
- job.setStartMethod(job.START_DISABLE);
- job.setHopcountMode(job.HOPCOUNT_ACCURATE);
- job.addHopCountFilter("child",new Long(2));
-
- // Now, set up the document specification.
- DocumentSpecification ds = job.getSpecification();
- // Crawl everything underneath the 'testdata' area
- File testDataFile = new File("testdata").getCanonicalFile();
- if (!testDataFile.exists())
- throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
- if (!testDataFile.isDirectory())
- throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
- SpecificationNode sn = new SpecificationNode("startpoint");
- sn.setAttribute("path",testDataFile.toString());
- SpecificationNode n = new SpecificationNode("include");
- n.setAttribute("type","file");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- n = new SpecificationNode("include");
- n.setAttribute("type","directory");
- n.setAttribute("match","*");
- sn.addChild(sn.getChildCount(),n);
- ds.addChild(ds.getChildCount(),sn);
-
- // Set up the output specification.
- OutputSpecification os = job.getOutputSpecification();
- // Null output connections have no output specification, so this is a no-op.
-
- // Save the job.
- jobManager.save(job);
-
- // Create the test data files.
- createFile(new File("testdata/test1.txt"),"This is a test file");
- createFile(new File("testdata/test2.txt"),"This is another test file");
- createDirectory(new File("testdata/testdir"));
- createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
- createDirectory(new File("testdata/testdir/seconddir"));
- createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
-
- // Now, start the job, and wait until it completes.
- jobManager.manualStart(job.getID());
- waitJobInactive(jobManager,job.getID(),120000L);
-
- // Check to be sure we actually processed the right number of documents.
- JobStatus status = jobManager.getStatus(job.getID());
- // The test data area has 4 documents and 2 directories and we have to count the root directory too.
- // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7.
- if (status.getDocumentsProcessed() != 6)
- throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
-
- // Now, delete the job.
- jobManager.deleteJob(job.getID());
- waitJobDeleted(jobManager,job.getID(), 120000L);
-
- // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
- }
- catch (Exception e)
- {
- e.printStackTrace();
- throw e;
- }
+ tester.executeTest();
}
- protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
- int statusValue = status.getStatus();
- switch (statusValue)
- {
- case JobStatus.JOBSTATUS_NOTYETRUN:
- throw new ManifoldCFException("Job was never started.");
- case JobStatus.JOBSTATUS_COMPLETED:
- break;
- case JobStatus.JOBSTATUS_ERROR:
- throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
- default:
- ManifoldCF.sleep(1000L);
- continue;
- }
- return;
- }
- throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
- protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
- throws ManifoldCFException, InterruptedException
- {
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() < startTime + maxTime)
- {
- JobStatus status = jobManager.getStatus(jobID);
- if (status == null)
- return;
- ManifoldCF.sleep(1000L);
- }
- throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
- }
-
-
}