You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/12/21 04:10:05 UTC
svn commit: r1221585 [2/3] - in /incubator/lcf/trunk: ./
connectors/activedirectory/ connectors/alfresco/ connectors/cmis/
connectors/documentum/ connectors/filenet/ connectors/filesystem/
connectors/filesystem/connector/src/test/java/org/apache/manifo...
Propchange: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseMySQL.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseMySQL.java (added)
+++ incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseMySQL.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,120 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a testing base class that is responsible for setting up/tearing down the agents framework. */
+public class BaseMySQL extends org.apache.manifoldcf.agents.tests.BaseMySQL
+{
+
+ @Before
+ public void setUp()
+ throws Exception
+ {
+ try
+ {
+ localCleanUp();
+ }
+ catch (Exception e)
+ {
+ System.out.println("Warning: Preclean failed: "+e.getMessage());
+ }
+ try
+ {
+ localSetUp();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void localSetUp()
+ throws Exception
+ {
+ super.localSetUp();
+
+ // Install the agents tables
+ initialize();
+ ManifoldCF.initializeEnvironment();
+ IThreadContext tc = ThreadContextFactory.make();
+ IAgentManager mgr = AgentManagerFactory.make(tc);
+ mgr.registerAgent("org.apache.manifoldcf.crawler.system.CrawlerAgent");
+ }
+
+ @After
+ public void cleanUp()
+ throws Exception
+ {
+ try
+ {
+ localCleanUp();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void localCleanUp()
+ throws Exception
+ {
+ initialize();
+ if (isInitialized())
+ {
+ // Test the uninstall
+ ManifoldCF.initializeEnvironment();
+ IThreadContext tc = ThreadContextFactory.make();
+
+ Exception currentException = null;
+ try
+ {
+ IAgentManager mgr = AgentManagerFactory.make(tc);
+ mgr.unregisterAgent("org.apache.manifoldcf.crawler.system.CrawlerAgent");
+ }
+ catch (Exception e)
+ {
+ if (currentException != null)
+ currentException = e;
+ }
+ try
+ {
+ super.localCleanUp();
+ }
+ catch (Exception e)
+ {
+ if (currentException != null)
+ currentException = e;
+ }
+ if (currentException != null)
+ throw currentException;
+ }
+ }
+
+}
Propchange: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseMySQL.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectorBaseMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectorBaseMySQL.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectorBaseMySQL.java (added)
+++ incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectorBaseMySQL.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,330 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.authorities.interfaces.*;
+import org.apache.manifoldcf.core.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a testing base class that is responsible for setting up/tearing down the agents framework. */
+public class ConnectorBaseMySQL extends org.apache.manifoldcf.crawler.tests.BaseMySQL
+{
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[0];
+ }
+
+ protected String[] getConnectorNames()
+ {
+ return new String[0];
+ }
+
+ protected String[] getAuthorityClasses()
+ {
+ return new String[0];
+ }
+
+ protected String[] getAuthorityNames()
+ {
+ return new String[0];
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[0];
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[0];
+ }
+
+ @Before
+ public void setUp()
+ throws Exception
+ {
+ try
+ {
+ localCleanUp();
+ }
+ catch (Exception e)
+ {
+ System.out.println("Warning: Preclean failed: "+e.getMessage());
+ }
+ try
+ {
+ localSetUp();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void localSetUp()
+ throws Exception
+ {
+
+ super.localSetUp();
+
+ // Register the connector we're testing
+ initialize();
+ ManifoldCF.initializeEnvironment();
+ IThreadContext tc = ThreadContextFactory.make();
+
+ IDBInterface database = DBInterfaceFactory.make(tc,
+ ManifoldCF.getMasterDatabaseName(),
+ ManifoldCF.getMasterDatabaseUsername(),
+ ManifoldCF.getMasterDatabasePassword());
+
+ IConnectorManager mgr = ConnectorManagerFactory.make(tc);
+ IAuthorityConnectorManager authMgr = AuthorityConnectorManagerFactory.make(tc);
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
+ IOutputConnectorManager outputMgr = OutputConnectorManagerFactory.make(tc);
+ IOutputConnectionManager outputConnManager = OutputConnectionManagerFactory.make(tc);
+
+ // Deregistration should be done in a transaction
+ database.beginTransaction();
+ try
+ {
+ int i;
+
+ String[] connectorClasses = getConnectorClasses();
+ String[] connectorNames = getConnectorNames();
+
+ i = 0;
+ while (i < connectorClasses.length)
+ {
+ // First, register connector
+ mgr.registerConnector(connectorNames[i],connectorClasses[i]);
+ // Then, signal to all jobs that might depend on this connector that they can switch state
+ // Find the connection names that come with this class
+ String[] connectionNames = connManager.findConnectionsForConnector(connectorClasses[i]);
+ // For each connection name, modify the jobs to note that the connector is now installed
+ jobManager.noteConnectorRegistration(connectionNames);
+ i++;
+ }
+
+ String[] authorityClasses = getAuthorityClasses();
+ String[] authorityNames = getAuthorityNames();
+
+ i = 0;
+ while (i < authorityClasses.length)
+ {
+ authMgr.registerConnector(authorityNames[i],authorityClasses[i]);
+ i++;
+ }
+
+ String[] outputClasses = getOutputClasses();
+ String[] outputNames = getOutputNames();
+
+ i = 0;
+ while (i < outputClasses.length)
+ {
+ // First, register connector
+ outputMgr.registerConnector(outputNames[i],outputClasses[i]);
+ // Then, signal to all jobs that might depend on this connector that they can switch state
+ // Find the connection names that come with this class
+ String[] connectionNames = outputConnManager.findConnectionsForConnector(outputClasses[i]);
+ // For all connection names, notify all agents of the registration
+ AgentManagerFactory.noteOutputConnectorRegistration(tc,connectionNames);
+ i++;
+ }
+
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ }
+
+ @After
+ public void cleanUp()
+ throws Exception
+ {
+ try
+ {
+ localCleanUp();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void localCleanUp()
+ throws Exception
+ {
+ initialize();
+ if (isInitialized())
+ {
+ // Test the uninstall
+ ManifoldCF.initializeEnvironment();
+ IThreadContext tc = ThreadContextFactory.make();
+
+ Exception currentException = null;
+ // First, tear down all jobs, connections, authority connections, and output connections.
+ try
+ {
+ IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(tc);
+ IAuthorityConnectionManager authConnMgr = AuthorityConnectionManagerFactory.make(tc);
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+
+ // Now, get a list of the repository connections
+ IRepositoryConnection[] connections = connMgr.getAllConnections();
+ int i = 0;
+ while (i < connections.length)
+ {
+ connMgr.delete(connections[i++].getName());
+ }
+
+ // Get a list of authority connections
+ IAuthorityConnection[] authorities = authConnMgr.getAllConnections();
+ i = 0;
+ while (i < authorities.length)
+ {
+ authConnMgr.delete(authorities[i++].getName());
+ }
+
+ // Finally, get rid of output connections
+ IOutputConnection[] outputs = outputMgr.getAllConnections();
+ i = 0;
+ while (i < outputs.length)
+ {
+ outputMgr.delete(outputs[i++].getName());
+ }
+
+ }
+ catch (Exception e)
+ {
+ currentException = e;
+ }
+ try
+ {
+ IDBInterface database = DBInterfaceFactory.make(tc,
+ ManifoldCF.getMasterDatabaseName(),
+ ManifoldCF.getMasterDatabaseUsername(),
+ ManifoldCF.getMasterDatabasePassword());
+
+ IConnectorManager mgr = ConnectorManagerFactory.make(tc);
+ IAuthorityConnectorManager authMgr = AuthorityConnectorManagerFactory.make(tc);
+ IOutputConnectorManager outputMgr = OutputConnectorManagerFactory.make(tc);
+ IOutputConnectionManager outputConnManager = OutputConnectionManagerFactory.make(tc);
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
+
+ // Deregistration should be done in a transaction
+ database.beginTransaction();
+ try
+ {
+ int i;
+
+ String[] connectorClasses = getConnectorClasses();
+
+ i = 0;
+ while (i < connectorClasses.length)
+ {
+ // Find the connection names that come with this class
+ String[] connectionNames = connManager.findConnectionsForConnector(connectorClasses[i]);
+ // For each connection name, modify the jobs to note that the connector is no longer installed
+ jobManager.noteConnectorDeregistration(connectionNames);
+ // Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
+ mgr.unregisterConnector(connectorClasses[i]);
+ i++;
+ }
+
+ String[] authorityClasses = getAuthorityClasses();
+
+ i = 0;
+ while (i < authorityClasses.length)
+ {
+ authMgr.unregisterConnector(authorityClasses[i]);
+ i++;
+ }
+
+ String[] outputClasses = getOutputClasses();
+
+ i = 0;
+ while (i < outputClasses.length)
+ {
+ // Find the connection names that come with this class
+ String[] connectionNames = outputConnManager.findConnectionsForConnector(outputClasses[i]);
+ // For all connection names, notify all agents of the deregistration
+ AgentManagerFactory.noteOutputConnectorDeregistration(tc,connectionNames);
+ // Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
+ outputMgr.unregisterConnector(outputClasses[i]);
+ i++;
+ }
+
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ }
+ catch (Exception e)
+ {
+ if (currentException != null)
+ currentException = e;
+ }
+ try
+ {
+ super.localCleanUp();
+ }
+ catch (Exception e)
+ {
+ if (currentException != null)
+ currentException = e;
+ }
+ if (currentException != null)
+ throw currentException;
+ }
+ }
+
+}
Propchange: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectorBaseMySQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectorBaseMySQL.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SanityMySQLTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SanityMySQLTest.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SanityMySQLTest.java (added)
+++ incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SanityMySQLTest.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,41 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.agents.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class SanityMySQLTest extends BaseMySQL
+{
+
+ @Test
+ public void sanityCheck()
+ throws Exception
+ {
+ // If we get this far, it must mean that the setup was successful, which is all that I'm shooting for in this test.
+ }
+
+
+}
Propchange: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SanityMySQLTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SanityMySQLTest.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BaseMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BaseMySQL.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BaseMySQL.java (added)
+++ incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BaseMySQL.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,523 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_loadtests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.mortbay.log.Logger;
+
+import org.apache.commons.httpclient.*;
+import org.apache.commons.httpclient.methods.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseMySQL extends org.apache.manifoldcf.crawler.tests.ConnectorBaseMySQL
+{
+ public static final String agentShutdownSignal = "agent-process";
+ public static final int testPort = 8346;
+
+ protected DaemonThread daemonThread = null;
+ protected Server server = null;
+
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+
+ protected void createDirectory(File f)
+ throws Exception
+ {
+ if (f.mkdirs() == false)
+ throw new Exception("Failed to create directory "+f.toString());
+ }
+
+ protected void removeDirectory(File f)
+ throws Exception
+ {
+ File[] files = f.listFiles();
+ if (files != null)
+ {
+ int i = 0;
+ while (i < files.length)
+ {
+ File subfile = files[i++];
+ if (subfile.isDirectory())
+ removeDirectory(subfile);
+ else
+ subfile.delete();
+ }
+ }
+ f.delete();
+ }
+
+ protected void createFile(File f, String contents)
+ throws Exception
+ {
+ OutputStream os = new FileOutputStream(f);
+ try
+ {
+ Writer w = new OutputStreamWriter(os,"utf-8");
+ try
+ {
+ w.write(contents);
+ }
+ finally
+ {
+ w.flush();
+ }
+ }
+ finally
+ {
+ os.close();
+ }
+ }
+
+ protected void removeFile(File f)
+ throws Exception
+ {
+ if (f.delete() == false)
+ throw new Exception("Failed to delete file "+f.toString());
+ }
+
+ protected void changeFile(File f, String newContents)
+ throws Exception
+ {
+ removeFile(f);
+ createFile(f,newContents);
+ }
+
+ // API support
+
+ // These methods allow communication with the ManifoldCF api webapp, via the locally-instantiated jetty
+
+ /** Construct a command url.
+ */
+ protected String makeAPIURL(String command)
+ {
+ return "http://localhost:"+Integer.toString(testPort)+"/mcf-api-service/json/"+command;
+ }
+
+ /** Perform an json API GET operation.
+ *@param apiURL is the operation.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIGetOperation(String apiURL, int expectedResponse)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ GetMethod method = new GetMethod(apiURL);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform an json API DELETE operation.
+ *@param apiURL is the operation.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIDeleteOperation(String apiURL, int expectedResponse)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ DeleteMethod method = new DeleteMethod(apiURL);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform an json API PUT operation.
+ *@param apiURL is the operation.
+ *@param input is the input JSON.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIPutOperation(String apiURL, int expectedResponse, String input)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ PutMethod method = new PutMethod(apiURL);
+ method.setRequestHeader("Content-type", "text/plain; charset=UTF-8");
+ method.setRequestBody(input);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform an json API POST operation.
+ *@param apiURL is the operation.
+ *@param input is the input JSON.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIPostOperation(String apiURL, int expectedResponse, String input)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ PostMethod method = new PostMethod(apiURL);
+ method.setRequestHeader("Content-type", "text/plain; charset=UTF-8");
+ method.setRequestBody(input);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform a json GET API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIGetOperationViaNodes(String command, int expectedResponse)
+ throws Exception
+ {
+ String result = performAPIGetOperation(makeAPIURL(command),expectedResponse);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ /** Perform a json DELETE API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIDeleteOperationViaNodes(String command, int expectedResponse)
+ throws Exception
+ {
+ String result = performAPIDeleteOperation(makeAPIURL(command),expectedResponse);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ /** Perform a json PUT API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIPutOperationViaNodes(String command, int expectedResponse, Configuration argument)
+ throws Exception
+ {
+ String argumentJson;
+ if (argument != null)
+ argumentJson = argument.toJSON();
+ else
+ argumentJson = null;
+
+ String result = performAPIPutOperation(makeAPIURL(command),expectedResponse,argumentJson);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ /** Perform a json POST API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIPostOperationViaNodes(String command, int expectedResponse, Configuration argument)
+ throws Exception
+ {
+ String argumentJson;
+ if (argument != null)
+ argumentJson = argument.toJSON();
+ else
+ argumentJson = null;
+
+ String result = performAPIPostOperation(makeAPIURL(command),expectedResponse,argumentJson);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ // Setup/teardown
+
+ @Before
+ public void setUp()
+ throws Exception
+ {
+ super.setUp();
+ // Start jetty
+ server = new Server( testPort );
+ server.setStopAtShutdown( true );
+
+
+ String crawlerWarPath = "../../framework/build/war/mcf-crawler-ui.war";
+ String authorityserviceWarPath = "../../framework/build/war/mcf-authority-service.war";
+ String apiWarPath = "../../framework/build/war/mcf-api-service.war";
+
+ if (System.getProperty("crawlerWarPath") != null)
+ crawlerWarPath = System.getProperty("crawlerWarPath");
+ if (System.getProperty("authorityserviceWarPath") != null)
+ authorityserviceWarPath = System.getProperty("authorityserviceWarPath");
+ if (System.getProperty("apiWarPath") != null)
+ apiWarPath = System.getProperty("apiWarPath");
+
+ // Initialize the servlets
+ WebAppContext lcfCrawlerUI = new WebAppContext(crawlerWarPath,"/mcf-crawler-ui");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfCrawlerUI.setParentLoaderPriority(true);
+ server.addHandler(lcfCrawlerUI);
+ WebAppContext lcfAuthorityService = new WebAppContext(authorityserviceWarPath,"/mcf-authority-service");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfAuthorityService.setParentLoaderPriority(true);
+ server.addHandler(lcfAuthorityService);
+ WebAppContext lcfApi = new WebAppContext(apiWarPath,"/mcf-api-service");
+ lcfApi.setParentLoaderPriority(true);
+ server.addHandler(lcfApi);
+ server.start();
+
+ // If all worked, then we can start the daemon.
+ // Clear the agents shutdown signal.
+ IThreadContext tc = ThreadContextFactory.make();
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ lockManager.clearGlobalFlag(agentShutdownSignal);
+
+ daemonThread = new DaemonThread();
+ daemonThread.start();
+ }
+
+ @After
+ public void cleanUp()
+ throws Exception
+ {
+ initialize();
+ if (isInitialized())
+ {
+ Exception currentException = null;
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Delete all jobs (and wait for them to go away)
+ if (daemonThread != null)
+ {
+ IJobManager jobManager = JobManagerFactory.make(tc);
+
+ // Get a list of the current active jobs
+ IJobDescription[] jobs = jobManager.getAllJobs();
+ int i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Abort this job, if it is running
+ try
+ {
+ jobManager.manualAbort(desc.getID());
+ }
+ catch (ManifoldCFException e)
+ {
+ // This generally means that the job was not running
+ }
+ }
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Wait for this job to stop
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(desc.getID(),false);
+ if (status != null)
+ {
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ case JobStatus.JOBSTATUS_COMPLETED:
+ case JobStatus.JOBSTATUS_ERROR:
+ break;
+ default:
+ ManifoldCF.sleep(10000);
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ // Now, delete them all
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ try
+ {
+ jobManager.deleteJob(desc.getID());
+ }
+ catch (ManifoldCFException e)
+ {
+ // This usually means that the job is already being deleted
+ }
+ }
+
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Wait for this job to disappear
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(desc.getID(),false);
+ if (status != null)
+ {
+ ManifoldCF.sleep(10000);
+ continue;
+ }
+ break;
+ }
+ }
+
+ // Shut down daemon
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ lockManager.setGlobalFlag(agentShutdownSignal);
+
+ // Wait for daemon thread to exit.
+ while (true)
+ {
+ if (daemonThread.isAlive())
+ {
+ Thread.sleep(1000L);
+ continue;
+ }
+ break;
+ }
+
+ Exception e = daemonThread.getDaemonException();
+ if (e != null)
+ currentException = e;
+ }
+
+ if (server != null)
+ {
+ server.stop();
+ server.join();
+ server = null;
+ }
+
+ // Clean up everything else
+ try
+ {
+ super.cleanUp();
+ }
+ catch (Exception e)
+ {
+ if (currentException == null)
+ currentException = e;
+ }
+ if (currentException != null)
+ throw currentException;
+ }
+ }
+
+ protected static class DaemonThread extends Thread
+ {
+ protected Exception daemonException = null;
+
+ public DaemonThread()
+ {
+ setName("Daemon thread");
+ }
+
+ public void run()
+ {
+ IThreadContext tc = ThreadContextFactory.make();
+ // Now, start the server, and then wait for the shutdown signal. On shutdown, we have to actually do the cleanup,
+ // because the JVM isn't going away.
+ try
+ {
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ while (true)
+ {
+ // Any shutdown signal yet?
+ if (lockManager.checkGlobalFlag(agentShutdownSignal))
+ break;
+
+ // Start whatever agents need to be started
+ ManifoldCF.startAgents(tc);
+
+ try
+ {
+ ManifoldCF.sleep(5000);
+ }
+ catch (InterruptedException e)
+ {
+ break;
+ }
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ daemonException = e;
+ }
+ finally
+ {
+ try
+ {
+ ManifoldCF.stopAgents(tc);
+ }
+ catch (ManifoldCFException e)
+ {
+ daemonException = e;
+ }
+ }
+ }
+
+ public Exception getDaemonException()
+ {
+ return daemonException;
+ }
+
+ }
+
+}
Propchange: incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BaseMySQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BaseMySQL.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BigCrawlMySQLTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BigCrawlMySQLTest.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BigCrawlMySQLTest.java (added)
+++ incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BigCrawlMySQLTest.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,240 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_loadtests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlMySQLTest extends BaseMySQL
+{
+
+ @Before
+ public void createTestArea()
+ throws Exception
+ {
+ try
+ {
+ File f = new File("testdata");
+ removeDirectory(f);
+ createDirectory(f);
+ // Create the test data files.
+ String baseFileName = "testdata/";
+ int i0 = 0;
+ while (i0 < 10)
+ {
+ String fileName0 = baseFileName + "/dir-" + i0;
+ createDirectory(new File(fileName0));
+ int i1 = 0;
+ while (i1 < 10)
+ {
+ String fileName1 = fileName0 + "/dir-" + i1;
+ createDirectory(new File(fileName1));
+ int i2 = 0;
+ while (i2 < 10)
+ {
+ String fileName2 = fileName1 + "/dir-" + i2;
+ createDirectory(new File(fileName2));
+ int i3 = 0;
+ while (i3 < 10)
+ {
+ String fileName3 = fileName2 + "/dir-" + i3;
+ createDirectory(new File(fileName3));
+ int i4 = 0;
+ while (i4 < 10)
+ {
+ String fileName4 = fileName3 + "/file-"+i4;
+ createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4);
+ i4++;
+ }
+ i3++;
+ }
+ i2++;
+ }
+ i1++;
+ }
+ i0++;
+ }
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ System.out.println("Done generating files");
+ }
+
+ @After
+ public void removeTestArea()
+ throws Exception
+ {
+ System.out.println("Removing generated files");
+ try
+ {
+ File f = new File("testdata");
+ removeDirectory(f);
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ try
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ long startTime = System.currentTimeMillis();
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID(),18000000L);
+ System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 111111)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ waitJobDeleted(jobManager,job.getID(),18000000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+ throws ManifoldCFException, InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ throw new ManifoldCFException("No such job: '"+jobID+"'");
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ throw new ManifoldCFException("Job was never started.");
+ case JobStatus.JOBSTATUS_COMPLETED:
+ break;
+ case JobStatus.JOBSTATUS_ERROR:
+ throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+ default:
+ ManifoldCF.sleep(1000L);
+ continue;
+ }
+ return;
+ }
+ throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+ protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+ throws ManifoldCFException, InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ return;
+ ManifoldCF.sleep(1000L);
+ }
+ throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+
+}
Propchange: incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BigCrawlMySQLTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/loadtests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_loadtests/BigCrawlMySQLTest.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseMySQL.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseMySQL.java (added)
+++ incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseMySQL.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,461 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.rss_loadtests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.mortbay.log.Logger;
+
+import org.apache.commons.httpclient.*;
+import org.apache.commons.httpclient.methods.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class BaseMySQL extends org.apache.manifoldcf.crawler.tests.ConnectorBaseMySQL
+{
+ public static final String agentShutdownSignal = "agent-process";
+ public static final int testPort = 8346;
+
+ protected DaemonThread daemonThread = null;
+ protected Server server = null;
+
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"File Connector"};
+ }
+
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.connectors.rss.RSSConnector"};
+ }
+
+ protected String[] getOutputNames()
+ {
+ return new String[]{"Null Output"};
+ }
+
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+ }
+
+ // API support
+
+ // These methods allow communication with the ManifoldCF api webapp, via the locally-instantiated jetty
+
+ /** Construct a command url.
+ */
+ protected String makeAPIURL(String command)
+ {
+ return "http://localhost:"+Integer.toString(testPort)+"/mcf-api-service/json/"+command;
+ }
+
+ /** Perform an json API GET operation.
+ *@param apiURL is the operation.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIGetOperation(String apiURL, int expectedResponse)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ GetMethod method = new GetMethod(apiURL);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform an json API DELETE operation.
+ *@param apiURL is the operation.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIDeleteOperation(String apiURL, int expectedResponse)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ DeleteMethod method = new DeleteMethod(apiURL);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform an json API PUT operation.
+ *@param apiURL is the operation.
+ *@param input is the input JSON.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIPutOperation(String apiURL, int expectedResponse, String input)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ PutMethod method = new PutMethod(apiURL);
+ method.setRequestHeader("Content-type", "text/plain; charset=UTF-8");
+ method.setRequestBody(input);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform an json API POST operation.
+ *@param apiURL is the operation.
+ *@param input is the input JSON.
+ *@param expectedResponse is the expected response code.
+ *@return the json response.
+ */
+ protected String performAPIPostOperation(String apiURL, int expectedResponse, String input)
+ throws Exception
+ {
+ HttpClient client = new HttpClient();
+ PostMethod method = new PostMethod(apiURL);
+ method.setRequestHeader("Content-type", "text/plain; charset=UTF-8");
+ method.setRequestBody(input);
+ int response = client.executeMethod(method);
+ byte[] responseData = method.getResponseBody();
+ String responseString = new String(responseData,"utf-8");
+ if (response != expectedResponse)
+ throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+ // We presume that the data is utf-8, since that's what the API uses throughout.
+ return responseString;
+ }
+
+ /** Perform a json GET API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIGetOperationViaNodes(String command, int expectedResponse)
+ throws Exception
+ {
+ String result = performAPIGetOperation(makeAPIURL(command),expectedResponse);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ /** Perform a json DELETE API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIDeleteOperationViaNodes(String command, int expectedResponse)
+ throws Exception
+ {
+ String result = performAPIDeleteOperation(makeAPIURL(command),expectedResponse);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ /** Perform a json PUT API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIPutOperationViaNodes(String command, int expectedResponse, Configuration argument)
+ throws Exception
+ {
+ String argumentJson;
+ if (argument != null)
+ argumentJson = argument.toJSON();
+ else
+ argumentJson = null;
+
+ String result = performAPIPutOperation(makeAPIURL(command),expectedResponse,argumentJson);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ /** Perform a json POST API operation, using Configuration structures to represent the json. This is for testing convenience,
+ * mostly.
+ */
+ protected Configuration performAPIPostOperationViaNodes(String command, int expectedResponse, Configuration argument)
+ throws Exception
+ {
+ String argumentJson;
+ if (argument != null)
+ argumentJson = argument.toJSON();
+ else
+ argumentJson = null;
+
+ String result = performAPIPostOperation(makeAPIURL(command),expectedResponse,argumentJson);
+ Configuration cfg = new Configuration();
+ cfg.fromJSON(result);
+ return cfg;
+ }
+
+ // Setup/teardown
+
+ @Before
+ public void setUp()
+ throws Exception
+ {
+ super.setUp();
+ // Start jetty
+ server = new Server( testPort );
+ server.setStopAtShutdown( true );
+
+
+ String crawlerWarPath = "../../framework/build/war/mcf-crawler-ui.war";
+ String authorityserviceWarPath = "../../framework/build/war/mcf-authority-service.war";
+ String apiWarPath = "../../framework/build/war/mcf-api-service.war";
+
+ if (System.getProperty("crawlerWarPath") != null)
+ crawlerWarPath = System.getProperty("crawlerWarPath");
+ if (System.getProperty("authorityserviceWarPath") != null)
+ authorityserviceWarPath = System.getProperty("authorityserviceWarPath");
+ if (System.getProperty("apiWarPath") != null)
+ apiWarPath = System.getProperty("apiWarPath");
+
+ // Initialize the servlets
+ WebAppContext lcfCrawlerUI = new WebAppContext(crawlerWarPath,"/mcf-crawler-ui");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfCrawlerUI.setParentLoaderPriority(true);
+ server.addHandler(lcfCrawlerUI);
+ WebAppContext lcfAuthorityService = new WebAppContext(authorityserviceWarPath,"/mcf-authority-service");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfAuthorityService.setParentLoaderPriority(true);
+ server.addHandler(lcfAuthorityService);
+ WebAppContext lcfApi = new WebAppContext(apiWarPath,"/mcf-api-service");
+ lcfApi.setParentLoaderPriority(true);
+ server.addHandler(lcfApi);
+ server.start();
+
+ // If all worked, then we can start the daemon.
+ // Clear the agents shutdown signal.
+ IThreadContext tc = ThreadContextFactory.make();
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ lockManager.clearGlobalFlag(agentShutdownSignal);
+
+ daemonThread = new DaemonThread();
+ daemonThread.start();
+ }
+
+ @After
+ public void cleanUp()
+ throws Exception
+ {
+ initialize();
+ if (isInitialized())
+ {
+ Exception currentException = null;
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Delete all jobs (and wait for them to go away)
+ if (daemonThread != null)
+ {
+ IJobManager jobManager = JobManagerFactory.make(tc);
+
+ // Get a list of the current active jobs
+ IJobDescription[] jobs = jobManager.getAllJobs();
+ int i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Abort this job, if it is running
+ try
+ {
+ jobManager.manualAbort(desc.getID());
+ }
+ catch (ManifoldCFException e)
+ {
+ // This generally means that the job was not running
+ }
+ }
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Wait for this job to stop
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(desc.getID(),false);
+ if (status != null)
+ {
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ case JobStatus.JOBSTATUS_COMPLETED:
+ case JobStatus.JOBSTATUS_ERROR:
+ break;
+ default:
+ ManifoldCF.sleep(10000);
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ // Now, delete them all
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ try
+ {
+ jobManager.deleteJob(desc.getID());
+ }
+ catch (ManifoldCFException e)
+ {
+ // This usually means that the job is already being deleted
+ }
+ }
+
+ i = 0;
+ while (i < jobs.length)
+ {
+ IJobDescription desc = jobs[i++];
+ // Wait for this job to disappear
+ while (true)
+ {
+ JobStatus status = jobManager.getStatus(desc.getID(),false);
+ if (status != null)
+ {
+ ManifoldCF.sleep(10000);
+ continue;
+ }
+ break;
+ }
+ }
+
+ // Shut down daemon
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ lockManager.setGlobalFlag(agentShutdownSignal);
+
+ // Wait for daemon thread to exit.
+ while (true)
+ {
+ if (daemonThread.isAlive())
+ {
+ Thread.sleep(1000L);
+ continue;
+ }
+ break;
+ }
+
+ Exception e = daemonThread.getDaemonException();
+ if (e != null)
+ currentException = e;
+ }
+
+ if (server != null)
+ {
+ server.stop();
+ server.join();
+ server = null;
+ }
+
+ // Clean up everything else
+ try
+ {
+ super.cleanUp();
+ }
+ catch (Exception e)
+ {
+ if (currentException == null)
+ currentException = e;
+ }
+ if (currentException != null)
+ throw currentException;
+ }
+ }
+
+ protected static class DaemonThread extends Thread
+ {
+ protected Exception daemonException = null;
+
+ public DaemonThread()
+ {
+ setName("Daemon thread");
+ }
+
+ public void run()
+ {
+ IThreadContext tc = ThreadContextFactory.make();
+ // Now, start the server, and then wait for the shutdown signal. On shutdown, we have to actually do the cleanup,
+ // because the JVM isn't going away.
+ try
+ {
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ while (true)
+ {
+ // Any shutdown signal yet?
+ if (lockManager.checkGlobalFlag(agentShutdownSignal))
+ break;
+
+ // Start whatever agents need to be started
+ ManifoldCF.startAgents(tc);
+
+ try
+ {
+ ManifoldCF.sleep(5000);
+ }
+ catch (InterruptedException e)
+ {
+ break;
+ }
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ daemonException = e;
+ }
+ finally
+ {
+ try
+ {
+ ManifoldCF.stopAgents(tc);
+ }
+ catch (ManifoldCFException e)
+ {
+ daemonException = e;
+ }
+ }
+ }
+
+ public Exception getDaemonException()
+ {
+ return daemonException;
+ }
+
+ }
+
+}
Propchange: incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseMySQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseMySQL.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlMySQLTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlMySQLTest.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlMySQLTest.java (added)
+++ incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlMySQLTest.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,186 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.rss_loadtests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import org.apache.manifoldcf.crawler.connectors.rss.RSSConnector;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class BigCrawlMySQLTest extends BaseMySQL
+{
+
+ protected MockRSSService rssService = null;
+
+ // Setup and teardown the mock wiki service
+
+ @Before
+ public void createRSSService()
+ throws Exception
+ {
+ rssService = new MockRSSService(10);
+ rssService.start();
+ }
+
+ @After
+ public void shutdownRSSService()
+ throws Exception
+ {
+ if (rssService != null)
+ rssService.stop();
+ }
+
+ @Test
+ public void bigCrawl()
+ throws Exception
+ {
+ try
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("RSS Connection");
+ conn.setDescription("RSS Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.connectors.rss.RSSConnector");
+ conn.setMaxConnections(100);
+ ConfigParams cp = conn.getConfigParams();
+ cp.setParameter(RSSConnector.emailParameter,"somebody@somewhere.com");
+ cp.setParameter(RSSConnector.maxOpenParameter,"100");
+ cp.setParameter(RSSConnector.maxFetchesParameter,"1000000");
+ cp.setParameter(RSSConnector.bandwidthParameter,"1000000");
+ cp.setParameter(RSSConnector.robotsUsageParameter,"none");
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("RSS Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // For 100000 documents, set up 10000 seeds
+ for (int i = 0 ; i < 10000 ; i++)
+ {
+ SpecificationNode sn = new SpecificationNode("feed");
+ sn.setAttribute("url","http://localhost:8189/rss/gen.php?type=feed&feed="+i);
+ ds.addChild(ds.getChildCount(),sn);
+ }
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Null output connections have no output specification, so this is a no-op.
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ long startTime = System.currentTimeMillis();
+ jobManager.manualStart(job.getID());
+ waitJobInactive(jobManager,job.getID(),22000000L);
+ System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 110000)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 110000, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ waitJobDeleted(jobManager,job.getID(),18000000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+ throws ManifoldCFException, InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ throw new ManifoldCFException("No such job: '"+jobID+"'");
+ int statusValue = status.getStatus();
+ switch (statusValue)
+ {
+ case JobStatus.JOBSTATUS_NOTYETRUN:
+ throw new ManifoldCFException("Job was never started.");
+ case JobStatus.JOBSTATUS_COMPLETED:
+ break;
+ case JobStatus.JOBSTATUS_ERROR:
+ throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+ default:
+ ManifoldCF.sleep(1000L);
+ continue;
+ }
+ return;
+ }
+ throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+ protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+ throws ManifoldCFException, InterruptedException
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ JobStatus status = jobManager.getStatus(jobID);
+ if (status == null)
+ return;
+ ManifoldCF.sleep(1000L);
+ }
+ throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+
+}
Propchange: incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlMySQLTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlMySQLTest.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/APISanityMySQLIT.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/APISanityMySQLIT.java?rev=1221585&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/APISanityMySQLIT.java (added)
+++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/APISanityMySQLIT.java Wed Dec 21 03:10:03 2011
@@ -0,0 +1,404 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class APISanityMySQLIT extends BaseMySQL
+{
+
+ @Before
+ public void createTestArea()
+ throws Exception
+ {
+ try
+ {
+ File f = new File("testdata");
+ removeDirectory(f);
+ createDirectory(f);
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @After
+ public void removeTestArea()
+ throws Exception
+ {
+ try
+ {
+ File f = new File("testdata");
+ removeDirectory(f);
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void sanityCheck()
+ throws Exception
+ {
+ try
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+ int i;
+ IJobManager jobManager = JobManagerFactory.make(tc);
+
+ // Create a basic file system connection, and save it.
+ ConfigurationNode connectionObject;
+ ConfigurationNode child;
+ Configuration requestObject;
+ Configuration result;
+
+ connectionObject = new ConfigurationNode("repositoryconnection");
+
+ child = new ConfigurationNode("name");
+ child.setValue("File Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("class_name");
+ child.setValue("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("description");
+ child.setValue("File Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("max_connections");
+ child.setValue("100");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ requestObject = new Configuration();
+ requestObject.addChild(0,connectionObject);
+
+ result = performAPIPutOperationViaNodes("repositoryconnections/File%20Connection",201,requestObject);
+
+ i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ }
+
+ // Create a basic null output connection, and save it.
+ connectionObject = new ConfigurationNode("outputconnection");
+
+ child = new ConfigurationNode("name");
+ child.setValue("Null Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("class_name");
+ child.setValue("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("description");
+ child.setValue("Null Connection");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ child = new ConfigurationNode("max_connections");
+ child.setValue("100");
+ connectionObject.addChild(connectionObject.getChildCount(),child);
+
+ requestObject = new Configuration();
+ requestObject.addChild(0,connectionObject);
+
+ result = performAPIPutOperationViaNodes("outputconnections/Null%20Connection",201,requestObject);
+
+ i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ }
+
+ // Create a job.
+ ConfigurationNode jobObject = new ConfigurationNode("job");
+
+ child = new ConfigurationNode("description");
+ child.setValue("Test Job");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("repository_connection");
+ child.setValue("File Connection");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("output_connection");
+ child.setValue("Null Connection");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("run_mode");
+ child.setValue("scan once");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("start_mode");
+ child.setValue("manual");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("hopcount_mode");
+ child.setValue("accurate");
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ child = new ConfigurationNode("document_specification");
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString());
+ ConfigurationNode sn = new ConfigurationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ ConfigurationNode n = new ConfigurationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new ConfigurationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ child.addChild(child.getChildCount(),sn);
+ jobObject.addChild(jobObject.getChildCount(),child);
+
+ requestObject = new Configuration();
+ requestObject.addChild(0,jobObject);
+
+ result = performAPIPostOperationViaNodes("jobs",201,requestObject);
+
+ String jobIDString = null;
+ i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ else if (resultNode.getType().equals("job_id"))
+ jobIDString = resultNode.getValue();
+ }
+ if (jobIDString == null)
+ throw new Exception("Missing job_id from return!");
+
+ Long jobID = new Long(jobIDString);
+
+ // Create the test data files.
+ createFile(new File("testdata/test1.txt"),"This is a test file");
+ createFile(new File("testdata/test2.txt"),"This is another test file");
+ createDirectory(new File("testdata/testdir"));
+ createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+
+ ConfigurationNode requestNode;
+
+ // Now, start the job, and wait until it completes.
+ startJob(jobIDString);
+ waitJobInactive(jobIDString, 120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ long count;
+ count = getJobDocumentsProcessed(jobIDString);
+ if (count != 5)
+ throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(count).toString());
+
+ // Add a file and recrawl
+ createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+ // Now, start the job, and wait until it completes.
+ startJob(jobIDString);
+ waitJobInactive(jobIDString, 120000L);
+
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ count = getJobDocumentsProcessed(jobIDString);
+ if (count != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(count).toString());
+
+ // Change a file, and recrawl
+ changeFile(new File("testdata/test1.txt"),"Modified contents");
+
+ // Now, start the job, and wait until it completes.
+ startJob(jobIDString);
+ waitJobInactive(jobIDString, 120000L);
+
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ count = getJobDocumentsProcessed(jobIDString);
+ if (count != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(count).toString());
+ // We also need to make sure the new document was indexed. Have to think about how to do this though.
+ // MHL
+
+ // Delete a file, and recrawl
+ removeFile(new File("testdata/test2.txt"));
+
+ // Now, start the job, and wait until it completes.
+ startJob(jobIDString);
+ waitJobInactive(jobIDString, 120000L);
+
+ // Check to be sure we actually processed the right number of documents.
+ // The test data area has 3 documents and one directory, and we have to count the root directory too.
+ count = getJobDocumentsProcessed(jobIDString);
+ if (count != 5)
+ throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(count).toString());
+
+ // Now, delete the job.
+ deleteJob(jobIDString);
+
+ waitJobDeleted(jobIDString, 120000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected void startJob(String jobIDString)
+ throws Exception
+ {
+ Configuration requestObject = new Configuration();
+
+ Configuration result = performAPIPutOperationViaNodes("start/"+jobIDString,201,requestObject);
+ int i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ }
+ }
+
+ protected void deleteJob(String jobIDString)
+ throws Exception
+ {
+ Configuration result = performAPIDeleteOperationViaNodes("jobs/"+jobIDString,200);
+ int i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ }
+
+ }
+
+ protected String getJobStatus(String jobIDString)
+ throws Exception
+ {
+ Configuration result = performAPIGetOperationViaNodes("jobstatusesnocounts/"+jobIDString,200);
+ String status = null;
+ int i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ else if (resultNode.getType().equals("jobstatus"))
+ {
+ int j = 0;
+ while (j < resultNode.getChildCount())
+ {
+ ConfigurationNode childNode = resultNode.findChild(j++);
+ if (childNode.getType().equals("status"))
+ status = childNode.getValue();
+ }
+ }
+ }
+ return status;
+ }
+
+ protected long getJobDocumentsProcessed(String jobIDString)
+ throws Exception
+ {
+ Configuration result = performAPIGetOperationViaNodes("jobstatuses/"+jobIDString,200);
+ String documentsProcessed = null;
+ int i = 0;
+ while (i < result.getChildCount())
+ {
+ ConfigurationNode resultNode = result.findChild(i++);
+ if (resultNode.getType().equals("error"))
+ throw new Exception(resultNode.getValue());
+ else if (resultNode.getType().equals("jobstatus"))
+ {
+ int j = 0;
+ while (j < resultNode.getChildCount())
+ {
+ ConfigurationNode childNode = resultNode.findChild(j++);
+ if (childNode.getType().equals("documents_processed"))
+ documentsProcessed = childNode.getValue();
+ }
+ }
+ }
+ if (documentsProcessed == null)
+ throw new Exception("Expected a documents_processed field, didn't find it");
+ return new Long(documentsProcessed).longValue();
+ }
+
+ protected void waitJobInactive(String jobIDString, long maxTime)
+ throws Exception
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ String status = getJobStatus(jobIDString);
+ if (status == null)
+ throw new Exception("No such job: '"+jobIDString+"'");
+ if (status.equals("not yet run"))
+ throw new Exception("Job was never started.");
+ if (status.equals("done"))
+ return;
+ if (status.equals("error"))
+ throw new Exception("Job reports error.");
+ ManifoldCF.sleep(1000L);
+ continue;
+ }
+ throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+ protected void waitJobDeleted(String jobIDString, long maxTime)
+ throws Exception
+ {
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() < startTime + maxTime)
+ {
+ String status = getJobStatus(jobIDString);
+ if (status == null)
+ return;
+ ManifoldCF.sleep(1000L);
+ }
+ throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+ }
+
+
+}
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/APISanityMySQLIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/APISanityMySQLIT.java
------------------------------------------------------------------------------
svn:keywords = Id