You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mi...@apache.org on 2013/06/24 05:26:58 UTC
svn commit: r1495913 [4/5] - in /manifoldcf/branches/CONNECTORS-728: ./
connectors/
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/
connectors/filesystem/connector/src/main/native2ascii/org/apache/mani...
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,56 @@
+/* $Id: APISanity.java 996524 2010-09-13 13:38:01Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class APISanityPostgresqlIT extends BasePostgresql
+{
+ protected APISanityTester tester;
+
+ public APISanityPostgresqlIT()
+ {
+ tester = new APISanityTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void sanityCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,266 @@
+/* $Id: APISanityTester.java 1449563 2013-02-24 22:52:28Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a very basic sanity check */
+public class APISanityTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public APISanityTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ int i;
+
+ // Create a basic file system connection, and save it.
+ ConfigurationNode connectionObject;
+ ConfigurationNode child;
+ Configuration requestObject;
+ Configuration result;
+
+ connectionObject = new ConfigurationNode("repositoryconnection");
+
+ child = new ConfigurationNode("name");
+ child.setValue("File Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("class_name");
+ child.setValue("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("description");
+ child.setValue("File Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("max_connections");
+ child.setValue("100");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ requestObject = new Configuration();
+ requestObject.addChild(0,connectionObject);
+
+ result = instance.performAPIPutOperationViaNodes("repositoryconnections/File%20Connection",201,requestObject);
+
+ i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ }
+
+ // Create a basic null output connection, and save it.
+ connectionObject = new ConfigurationNode("outputconnection");
+
+ child = new ConfigurationNode("name");
+ child.setValue("Null Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("class_name");
+ child.setValue("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("description");
+ child.setValue("Null Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("max_connections");
+ child.setValue("100");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ requestObject = new Configuration();
+ requestObject.addChild(0,connectionObject);
+
+ result = instance.performAPIPutOperationViaNodes("outputconnections/Null%20Connection",201,requestObject);
+
+ i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ }
+
+ // Create a job.
+ ConfigurationNode jobObject = new ConfigurationNode("job");
+
+ child = new ConfigurationNode("description");
+ child.setValue("Test Job");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("repository_connection");
+ child.setValue("File Connection");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("output_connection");
+ child.setValue("Null Connection");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("run_mode");
+ child.setValue("scan once");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("start_mode");
+ child.setValue("manual");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("hopcount_mode");
+ child.setValue("accurate");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("document_specification");
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ ConfigurationNode sn = new ConfigurationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ ConfigurationNode n = new ConfigurationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new ConfigurationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ child.addChild(child.getChildCount(),sn);
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ requestObject = new Configuration();
+ requestObject.addChild(0,jobObject);
+
+ result = instance.performAPIPostOperationViaNodes("jobs",201,requestObject);
+
+ String jobIDString = null;
+ i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ else if (resultNode.getType().equals("job_id"))
+ jobIDString = resultNode.getValue();
+ }
+ if (jobIDString == null)
+ throw new Exception("Missing job_id from return!");
+
+ Long jobID = new Long(jobIDString);
+
+ // Create the test data files.
+ FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file");
+ FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file");
+ FileHelper.createDirectory(new File("testdata/testdir"));
+ FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+
+ ConfigurationNode requestNode;
+
+ // Now, start the job, and wait until it completes.
+ instance.startJobAPI(jobIDString);
+ instance.waitJobInactiveAPI(jobIDString, 120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ long count;
+ count = instance.getJobDocumentsProcessedAPI(jobIDString);
+ if (count != 5)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(count).toString());
+
+ // Add a file and recrawl
+ FileHelper.createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+ // Now, start the job, and wait until it completes.
+ instance.startJobAPI(jobIDString);
+ instance.waitJobInactiveAPI(jobIDString, 120000L);
+
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ count = instance.getJobDocumentsProcessedAPI(jobIDString);
+ if (count != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(count).toString());
+
+ // Change a file, and recrawl
+ FileHelper.changeFile(new File("testdata/test1.txt"),"Modified contents");
+
+ // Now, start the job, and wait until it completes.
+ instance.startJobAPI(jobIDString);
+ instance.waitJobInactiveAPI(jobIDString, 120000L);
+
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ count = instance.getJobDocumentsProcessedAPI(jobIDString);
+ if (count != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(count).toString());
+ // We also need to make sure the new document was indexed. Have to think about how to do this though.
+ // MHL
+
+ // Delete a file, and recrawl
+ FileHelper.removeFile(new File("testdata/test2.txt"));
+
+ // Now, start the job, and wait until it completes.
+ instance.startJobAPI(jobIDString);
+ instance.waitJobInactiveAPI(jobIDString, 120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ count = instance.getJobDocumentsProcessedAPI(jobIDString);
+ if (count != 5)
+ throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(count).toString());
+
+ // Have a try to get the history records for the connection
+ result = instance.performAPIGetOperationViaNodes("repositoryconnectionhistory/File%20Connection?report=simple",200);
+
+ // Now, delete the job.
+ instance.deleteJobAPI(jobIDString);
+
+ instance.waitJobDeletedAPI(jobIDString, 120000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,52 @@
+/* $Id: TestBase.java 996524 2010-09-13 13:38:01Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseDerby extends org.apache.manifoldcf.crawler.tests.BaseITDerby
+{
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,62 @@
+/* $Id: BaseHSQLDB.java 1422189 2012-12-15 06:00:24Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseHSQLDB extends org.apache.manifoldcf.crawler.tests.BaseITHSQLDB
+{
+ public BaseHSQLDB()
+ {
+ super();
+ }
+
+ public BaseHSQLDB(boolean singleWar)
+ {
+ super(singleWar);
+ }
+
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,52 @@
+/* $Id: BaseMySQL.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseMySQL extends org.apache.manifoldcf.crawler.tests.BaseITMySQL
+{
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,52 @@
+/* $Id: TestBase.java 996524 2010-09-13 13:38:01Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BasePostgresql extends org.apache.manifoldcf.crawler.tests.BaseITPostgresql
+{
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,53 @@
+/* $Id: BaseUIDerby.java 1231798 2012-01-15 23:58:22Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseUIDerby extends org.apache.manifoldcf.crawler.tests.ConnectorBaseUIDerby
+{
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,63 @@
+/* $Id: BaseUIHSQLDB.java 1422192 2012-12-15 06:23:50Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseUIHSQLDB extends org.apache.manifoldcf.crawler.tests.ConnectorBaseUIHSQLDB
+{
+ public BaseUIHSQLDB()
+ {
+ super();
+ }
+
+ public BaseUIHSQLDB(boolean singleWar)
+ {
+ super(singleWar);
+ }
+
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: BigCrawlDerbyLT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlDerbyLT extends BaseDerby
+{
+ protected BigCrawlTester tester;
+
+ public BigCrawlDerbyLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: BigCrawlHSQLDBLT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlHSQLDBLT extends BaseHSQLDB
+{
+ protected BigCrawlTester tester;
+
+ public BigCrawlHSQLDBLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: BigCrawlMySQLLT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlMySQLLT extends BaseMySQL
+{
+ protected BigCrawlTester tester;
+
+ public BigCrawlMySQLLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: BigCrawlPostgresqlLT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlPostgresqlLT extends BasePostgresql
+{
+ protected BigCrawlTester tester;
+
+ public BigCrawlPostgresqlLT()
+ {
+ tester = new BigCrawlTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,176 @@
+/* $Id: BigCrawlTester.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a 100000 document crawl */
+public class BigCrawlTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public BigCrawlTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ // Create the test data files.
+ String baseFileName = "testdata/";
+ int i0 = 0;
+ while (i0 < 10)
+ {
+ String fileName0 = baseFileName + "/dir-" + i0;
+ FileHelper.createDirectory(new File(fileName0));
+ int i1 = 0;
+ while (i1 < 10)
+ {
+ String fileName1 = fileName0 + "/dir-" + i1;
+ FileHelper.createDirectory(new File(fileName1));
+ int i2 = 0;
+ while (i2 < 10)
+ {
+ String fileName2 = fileName1 + "/dir-" + i2;
+ FileHelper.createDirectory(new File(fileName2));
+ int i3 = 0;
+ while (i3 < 10)
+ {
+ String fileName3 = fileName2 + "/dir-" + i3;
+ FileHelper.createDirectory(new File(fileName3));
+ int i4 = 0;
+ while (i4 < 10)
+ {
+ String fileName4 = fileName3 + "/file-"+i4;
+ FileHelper.createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4);
+ i4++;
+ }
+ i3++;
+ }
+ i2++;
+ }
+ i1++;
+ }
+ i0++;
+ }
+ System.err.println("Done generating files");
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ System.err.println("Removing generated files");
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ long startTime = System.currentTimeMillis();
+ jobManager.manualStart(job.getID());
+ instance.waitJobInactiveNative(jobManager,job.getID(),18000000L);
+ System.err.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 111111)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),18000000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: ExpirationDerbyIT.java 1231798 2012-01-15 23:58:22Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationDerbyIT extends BaseDerby
+{
+ protected ExpirationTester tester;
+
+ public ExpirationDerbyIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void expirationCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,56 @@
+/* $Id: ExpirationHSQLDBIT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationHSQLDBIT extends BaseHSQLDB
+{
+ protected ExpirationTester tester;
+
+ public ExpirationHSQLDBIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void expirationCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,60 @@
+/* $Id: ExpirationMySQLIT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationMySQLIT extends BaseMySQL
+{
+ protected ExpirationTester tester;
+
+ public ExpirationMySQLIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void expirationCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: ExpirationPostgresqlIT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationPostgresqlIT extends BasePostgresql
+{
+ protected ExpirationTester tester;
+
+ public ExpirationPostgresqlIT()
+ {
+ tester = new ExpirationTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void expirationCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,166 @@
+/* $Id: ExpirationTester.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public ExpirationTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+ // 60-second expiration for all documents..
+ job.setExpiration(new Long(60000L));
+ // Infinite rescan interval.
+ job.setInterval(null);
+ // Continuous job.
+ job.setType(IJobDescription.TYPE_CONTINUOUS);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Create the test data files.
+ FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file");
+ FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file");
+ FileHelper.createDirectory(new File("testdata/testdir"));
+ FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+ FileHelper.createDirectory(new File("testdata/testdir/seconddir"));
+ FileHelper.createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
+
+ // Now, start the job, and wait until it is running.
+ jobManager.manualStart(job.getID());
+ instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+
+ JobStatus status;
+
+ // Now we wait, and we should see 7 documents eventually.
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + 120000L)
+ {
+ status = jobManager.getStatus(job.getID());
+ if (status.getDocumentsProcessed() == 7)
+ break;
+
+ ManifoldCF.sleep(1000L);
+ }
+
+ // At this point there should be 7 documents.
+ // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time.
+ // So all we need to do is confirm that the job stops within 2 minutes.
+ instance.waitJobInactiveNative(jobManager,job.getID(),180000L);
+
+ status = jobManager.getStatus(job.getID());
+ if (status.getDocumentsProcessed() != 0)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),120000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,90 @@
+/* $Id: FileHelper.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+
+/** Helper class for local file manipulation */
+public class FileHelper
+{
+
+ protected static void createDirectory(File f)
+ throws Exception
+ {
+ if (f.mkdirs() == false)
+ throw new Exception("Failed to create directory "+f.toString());
+ }
+
+ protected static void removeDirectory(File f)
+ throws Exception
+ {
+ File[] files = f.listFiles();
+ if (files != null)
+ {
+ int i = 0;
+ while (i < files.length)
+ {
+ File subfile = files[i++];
+ if (subfile.isDirectory())
+ removeDirectory(subfile);
+ else
+ subfile.delete();
+ }
+ }
+ f.delete();
+ }
+
+ protected static void createFile(File f, String contents)
+ throws Exception
+ {
+ OutputStream os = new FileOutputStream(f);
+ try
+ {
+ Writer w = new OutputStreamWriter(os,"utf-8");
+ try
+ {
+ w.write(contents);
+ }
+ finally
+ {
+ w.flush();
+ }
+ }
+ finally
+ {
+ os.close();
+ }
+ }
+
+ protected static void removeFile(File f)
+ throws Exception
+ {
+ if (f.delete() == false)
+ throw new Exception("Failed to delete file "+f.toString());
+ }
+
+ protected static void changeFile(File f, String newContents)
+ throws Exception
+ {
+ removeFile(f);
+ createFile(f,newContents);
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,56 @@
+/* $Id: HopcountDerbyIT.java 1231798 2012-01-15 23:58:22Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to be sure hopcount functionality is working properly. */
+public class HopcountDerbyIT extends BaseDerby
+{
+ protected HopcountTester tester;
+
+ public HopcountDerbyIT()
+ {
+ tester = new HopcountTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void hopcountCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,56 @@
+/* $Id: HopcountHSQLDBIT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to be sure hopcount functionality is working properly. */
+public class HopcountHSQLDBIT extends BaseHSQLDB
+{
+ protected HopcountTester tester;
+
+ public HopcountHSQLDBIT()
+ {
+ tester = new HopcountTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void hopcountCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: HopcountMySQLIT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to be sure hopcount functionality is working properly. */
+public class HopcountMySQLIT extends BaseMySQL
+{
+ protected HopcountTester tester;
+
+ public HopcountMySQLIT()
+ {
+ tester = new HopcountTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void hopcountCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,55 @@
+/* $Id: HopcountPostgresqlIT.java 1226141 2011-12-31 17:23:35Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to be sure hopcount functionality is working properly. */
+public class HopcountPostgresqlIT extends BasePostgresql
+{
+ protected HopcountTester tester;
+
+ public HopcountPostgresqlIT()
+ {
+ tester = new HopcountTester(mcfInstance);
+ }
+
+ @Before
+ public void setupTester()
+ throws Exception
+ {
+ tester.setupTestArea();
+ }
+
+ @After
+ public void teardownTester()
+ throws Exception
+ {
+ tester.teardownTestArea();
+ }
+
+ @Test
+ public void hopcountCheck()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,145 @@
+/* $Id: HopcountTester.java 1372225 2012-08-13 00:44:05Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a test which checks to be sure hopcount functionality is working properly. */
+public class HopcountTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public HopcountTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+ job.addHopCountFilter("child",new Long(2));
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Create the test data files.
+ FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file");
+ FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file");
+ FileHelper.createDirectory(new File("testdata/testdir"));
+ FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+ FileHelper.createDirectory(new File("testdata/testdir/seconddir"));
+ FileHelper.createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
+
+ // Now, start the job, and wait until it completes.
+ jobManager.manualStart(job.getID());
+ instance.waitJobInactiveNative(jobManager,job.getID(),120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 4 documents and 2 directories and we have to count the root directory too.
+ // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7.
+ if (status.getDocumentsProcessed() != 6)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(), 120000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java (added)
+++ manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,48 @@
+/* $Id: NavigationCombinedHSQLDBUI.java 1422222 2012-12-15 11:29:02Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+import org.apache.manifoldcf.core.tests.HTMLTester;
+
+/** Basic UI navigation tests */
+public class NavigationCombinedHSQLDBUI extends BaseUIHSQLDB
+{
+
+ public NavigationCombinedHSQLDBUI()
+ {
+ super(true);
+ }
+
+ @Test
+ public void createConnectionsAndJob()
+ throws Exception
+ {
+ new NavigationUITester(testerInstance,"http://localhost:8346/mcf/index.jsp").createConnectionsAndJob();
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java
------------------------------------------------------------------------------
svn:mime-type = text/plain