You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/01 23:13:56 UTC
svn commit: r1546865 - in /manifoldcf/branches/CONNECTORS-781/framework: ./
pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/
pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/
Author: kwright
Date: Sun Dec 1 22:13:56 2013
New Revision: 1546865
URL: http://svn.apache.org/r1546865
Log:
Add scheduling test.
Added:
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java (with props)
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java (with props)
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java (with props)
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java (with props)
Modified:
manifoldcf/branches/CONNECTORS-781/framework/build.xml
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITDerby.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITHSQLDB.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITMySQL.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITPostgresql.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
Modified: manifoldcf/branches/CONNECTORS-781/framework/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/build.xml?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/build.xml Sun Dec 1 22:13:56 2013
@@ -1486,6 +1486,25 @@
</junit>
</target>
+ <target name="run-pull-agent-tests" depends="compile-pull-agent,compile-pull-agent-tests">
+ <mkdir dir="test-output"/>
+ <junit fork="true" maxmemory="128m" dir="test-output" outputtoformatters="true" showoutput="true" haltonfailure="true">
+ <classpath>
+ <path refid="framework-classpath"/>
+ <pathelement location="build/core/classes"/>
+ <pathelement location="build/core-tests/classes"/>
+ <pathelement location="build/agents/classes"/>
+ <pathelement location="build/agents-tests/classes"/>
+ <pathelement location="build/pull-agent/classes"/>
+ <pathelement location="build/pull-agent-tests/classes"/>
+ </classpath>
+ <formatter type="brief" usefile="false"/>
+
+ <test name="org.apache.manifoldcf.crawler.tests.SchedulerHSQLDBTest" todir="test-output"/>
+
+ </junit>
+ </target>
+
<target name="run-script-engine-tests" depends="compile-core,compile-script-engine,compile-script-engine-tests">
<mkdir dir="test-output"/>
<junit fork="true" maxmemory="128m" dir="test-output" outputtoformatters="true" showoutput="true" haltonfailure="true">
@@ -1519,7 +1538,7 @@
</junit>
</target>
- <target name="run-tests" depends="compile-tests,run-core-tests,run-script-engine-tests"/>
+ <target name="run-tests" depends="compile-tests,run-core-tests,run-pull-agent-tests,run-script-engine-tests"/>
<target name="run-tests-derby" depends="compile-tests">
<mkdir dir="test-derby-output"/>
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connectors/BaseRepositoryConnector.java Sun Dec 1 22:13:56 2013
@@ -64,6 +64,7 @@ public abstract class BaseRepositoryConn
* This must return a model value as specified above.
*@return the model type value.
*/
+ @Override
public int getConnectorModel()
{
// Return the simplest model - full everything
@@ -73,6 +74,7 @@ public abstract class BaseRepositoryConn
/** Return the list of activities that this connector supports (i.e. writes into the log).
*@return the list.
*/
+ @Override
public String[] getActivitiesList()
{
return new String[0];
@@ -81,6 +83,7 @@ public abstract class BaseRepositoryConn
/** Return the list of relationship types that this connector recognizes.
*@return the list.
*/
+ @Override
public String[] getRelationshipTypes()
{
// The base situation is that there are no relationships.
@@ -97,6 +100,7 @@ public abstract class BaseRepositoryConn
*@return the set of bin names. If an empty array is returned, it is equivalent to there being no request
* rate throttling available for this identifier.
*/
+ @Override
public String[] getBinNames(String documentIdentifier)
{
// Base version has one bin for all documents. Use empty string for this since "*" would make
@@ -111,6 +115,7 @@ public abstract class BaseRepositoryConn
*@param command is the command, which is taken directly from the API request.
*@return true if the resource is found, false if not. In either case, output may be filled in.
*/
+ @Override
public boolean requestInfo(Configuration output, String command)
throws ManifoldCFException
{
@@ -143,6 +148,7 @@ public abstract class BaseRepositoryConn
*@param endTime is the end of the time range to consider, exclusive.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*/
+ @Override
public void addSeedDocuments(ISeedingActivity activities, DocumentSpecification spec,
long startTime, long endTime, int jobMode)
throws ManifoldCFException, ServiceInterruption
@@ -285,6 +291,7 @@ public abstract class BaseRepositoryConn
* Empty version strings indicate that there is no versioning ability for the corresponding document, and the document
* will always be processed.
*/
+ @Override
public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption
@@ -387,6 +394,7 @@ public abstract class BaseRepositoryConn
*@param documentIdentifiers is the set of document identifiers.
*@param versions is the corresponding set of version identifiers (individual identifiers may be null).
*/
+ @Override
public void releaseDocumentVersions(String[] documentIdentifiers, String[] versions)
throws ManifoldCFException
{
@@ -396,6 +404,7 @@ public abstract class BaseRepositoryConn
/** Get the maximum number of documents to amalgamate together into one batch, for this connector.
*@return the maximum number. 0 indicates "unlimited".
*/
+ @Override
public int getMaxDocumentRequest()
{
// Base implementation does one at a time.
@@ -416,6 +425,7 @@ public abstract class BaseRepositoryConn
* should only find other references, and should not actually call the ingestion methods.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*/
+ @Override
public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities,
DocumentSpecification spec, boolean[] scanOnly, int jobMode)
throws ManifoldCFException, ServiceInterruption
@@ -461,6 +471,7 @@ public abstract class BaseRepositoryConn
*@param ds is the current document specification for this job.
*@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
*/
+ @Override
public void outputSpecificationHeader(IHTTPOutput out, Locale locale, DocumentSpecification ds, List<String> tabsArray)
throws ManifoldCFException, IOException
{
@@ -502,6 +513,7 @@ public abstract class BaseRepositoryConn
*@param ds is the current document specification for this job.
*@param tabName is the current tab name.
*/
+ @Override
public void outputSpecificationBody(IHTTPOutput out, Locale locale, DocumentSpecification ds, String tabName)
throws ManifoldCFException, IOException
{
@@ -532,6 +544,7 @@ public abstract class BaseRepositoryConn
*@return null if all is well, or a string error message if there is an error that should prevent saving of
* the job (and cause a redirection to an error page).
*/
+ @Override
public String processSpecificationPost(IPostParameters variableContext, Locale locale, DocumentSpecification ds)
throws ManifoldCFException
{
@@ -561,6 +574,7 @@ public abstract class BaseRepositoryConn
*@param locale is the locale the output is preferred to be in.
*@param ds is the current document specification for this job.
*/
+ @Override
public void viewSpecification(IHTTPOutput out, Locale locale, DocumentSpecification ds)
throws ManifoldCFException, IOException
{
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITDerby.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITDerby.java?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITDerby.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITDerby.java Sun Dec 1 22:13:56 2013
@@ -100,6 +100,7 @@ public class BaseITDerby extends Connect
/** Construct a command url.
*/
protected String makeAPIURL(String command)
+ throws Exception
{
return mcfInstance.makeAPIURL(command);
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITHSQLDB.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITHSQLDB.java?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITHSQLDB.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITHSQLDB.java Sun Dec 1 22:13:56 2013
@@ -44,6 +44,12 @@ public class BaseITHSQLDB extends Connec
mcfInstance = new ManifoldCFInstance(singleWar);
}
+ public BaseITHSQLDB(boolean singleWar, boolean webapps)
+ {
+ super();
+ mcfInstance = new ManifoldCFInstance(singleWar, webapps);
+ }
+
// Basic job support
protected void waitJobInactiveNative(IJobManager jobManager, Long jobID, long maxTime)
@@ -101,6 +107,7 @@ public class BaseITHSQLDB extends Connec
/** Construct a command url.
*/
protected String makeAPIURL(String command)
+ throws Exception
{
return mcfInstance.makeAPIURL(command);
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITMySQL.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITMySQL.java?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITMySQL.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITMySQL.java Sun Dec 1 22:13:56 2013
@@ -101,6 +101,7 @@ public class BaseITMySQL extends Connect
/** Construct a command url.
*/
protected String makeAPIURL(String command)
+ throws Exception
{
return mcfInstance.makeAPIURL(command);
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITPostgresql.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITPostgresql.java?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITPostgresql.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/BaseITPostgresql.java Sun Dec 1 22:13:56 2013
@@ -101,6 +101,7 @@ public class BaseITPostgresql extends Co
/** Construct a command url.
*/
protected String makeAPIURL(String command)
+ throws Exception
{
return mcfInstance.makeAPIURL(command);
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java?rev=1546865&r1=1546864&r2=1546865&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java Sun Dec 1 22:13:56 2013
@@ -56,6 +56,7 @@ import org.apache.http.entity.ContentTyp
/** Tests that run the "agents daemon" should be derived from this */
public class ManifoldCFInstance
{
+ protected boolean webapps = true;
protected boolean singleWar = false;
protected int testPort = 8346;
@@ -68,16 +69,27 @@ public class ManifoldCFInstance
public ManifoldCFInstance(boolean singleWar)
{
- this(8346,singleWar);
+ this(8346,singleWar,true);
+ }
+
+ public ManifoldCFInstance(boolean singleWar, boolean webapps)
+ {
+ this(8346,singleWar,webapps);
}
public ManifoldCFInstance(int testPort)
{
- this(testPort,false);
+ this(testPort,false,true);
}
public ManifoldCFInstance(int testPort, boolean singleWar)
{
+ this(testPort,singleWar,true);
+ }
+
+ public ManifoldCFInstance(int testPort, boolean singleWar, boolean webapps)
+ {
+ this.webapps = webapps;
this.testPort = testPort;
this.singleWar = singleWar;
}
@@ -277,11 +289,17 @@ public class ManifoldCFInstance
/** Construct a command url.
*/
public String makeAPIURL(String command)
+ throws Exception
{
- if (singleWar)
- return "http://localhost:"+Integer.toString(testPort)+"/mcf/api/json/"+command;
+ if (webapps)
+ {
+ if (singleWar)
+ return "http://localhost:"+Integer.toString(testPort)+"/mcf/api/json/"+command;
+ else
+ return "http://localhost:"+Integer.toString(testPort)+"/mcf-api-service/json/"+command;
+ }
else
- return "http://localhost:"+Integer.toString(testPort)+"/mcf-api-service/json/"+command;
+ throw new Exception("No API servlet running");
}
public static String convertToString(HttpResponse httpResponse)
@@ -495,53 +513,64 @@ public class ManifoldCFInstance
public void start()
throws Exception
{
- // Start jetty
- server = new Server( testPort );
- server.setStopAtShutdown( true );
- // Initialize the servlets
ContextHandlerCollection contexts = new ContextHandlerCollection();
- server.setHandler(contexts);
+ if (webapps)
+ {
+ // Start jetty
+ server = new Server( testPort );
+ server.setStopAtShutdown( true );
+ // Initialize the servlets
+ server.setHandler(contexts);
+ }
if (singleWar)
{
- // Start the single combined war
- String combinedWarPath = "../../framework/build/war-proprietary/mcf-combined-service.war";
- if (System.getProperty("combinedWarPath") != null)
- combinedWarPath = System.getProperty("combinedWarPath");
-
- // Initialize the servlet
- WebAppContext lcfCombined = new WebAppContext(combinedWarPath,"/mcf");
- // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
- lcfCombined.setParentLoaderPriority(true);
- contexts.addHandler(lcfCombined);
- server.start();
+ if (webapps)
+ {
+ // Start the single combined war
+ String combinedWarPath = "../../framework/build/war-proprietary/mcf-combined-service.war";
+ if (System.getProperty("combinedWarPath") != null)
+ combinedWarPath = System.getProperty("combinedWarPath");
+
+ // Initialize the servlet
+ WebAppContext lcfCombined = new WebAppContext(combinedWarPath,"/mcf");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfCombined.setParentLoaderPriority(true);
+ contexts.addHandler(lcfCombined);
+ server.start();
+ }
+ else
+ throw new Exception("Can't run singleWar without webapps");
}
else
{
- String crawlerWarPath = "../../framework/build/war-proprietary/mcf-crawler-ui.war";
- String authorityserviceWarPath = "../../framework/build/war-proprietary/mcf-authority-service.war";
- String apiWarPath = "../../framework/build/war-proprietary/mcf-api-service.war";
-
- if (System.getProperty("crawlerWarPath") != null)
- crawlerWarPath = System.getProperty("crawlerWarPath");
- if (System.getProperty("authorityserviceWarPath") != null)
- authorityserviceWarPath = System.getProperty("authorityserviceWarPath");
- if (System.getProperty("apiWarPath") != null)
- apiWarPath = System.getProperty("apiWarPath");
-
- // Initialize the servlets
- WebAppContext lcfCrawlerUI = new WebAppContext(crawlerWarPath,"/mcf-crawler-ui");
- // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
- lcfCrawlerUI.setParentLoaderPriority(true);
- contexts.addHandler(lcfCrawlerUI);
- WebAppContext lcfAuthorityService = new WebAppContext(authorityserviceWarPath,"/mcf-authority-service");
- // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
- lcfAuthorityService.setParentLoaderPriority(true);
- contexts.addHandler(lcfAuthorityService);
- WebAppContext lcfApi = new WebAppContext(apiWarPath,"/mcf-api-service");
- lcfApi.setParentLoaderPriority(true);
- contexts.addHandler(lcfApi);
- server.start();
+ if (webapps)
+ {
+ String crawlerWarPath = "../../framework/build/war-proprietary/mcf-crawler-ui.war";
+ String authorityserviceWarPath = "../../framework/build/war-proprietary/mcf-authority-service.war";
+ String apiWarPath = "../../framework/build/war-proprietary/mcf-api-service.war";
+
+ if (System.getProperty("crawlerWarPath") != null)
+ crawlerWarPath = System.getProperty("crawlerWarPath");
+ if (System.getProperty("authorityserviceWarPath") != null)
+ authorityserviceWarPath = System.getProperty("authorityserviceWarPath");
+ if (System.getProperty("apiWarPath") != null)
+ apiWarPath = System.getProperty("apiWarPath");
+
+ // Initialize the servlets
+ WebAppContext lcfCrawlerUI = new WebAppContext(crawlerWarPath,"/mcf-crawler-ui");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfCrawlerUI.setParentLoaderPriority(true);
+ contexts.addHandler(lcfCrawlerUI);
+ WebAppContext lcfAuthorityService = new WebAppContext(authorityserviceWarPath,"/mcf-authority-service");
+ // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+ lcfAuthorityService.setParentLoaderPriority(true);
+ contexts.addHandler(lcfAuthorityService);
+ WebAppContext lcfApi = new WebAppContext(apiWarPath,"/mcf-api-service");
+ lcfApi.setParentLoaderPriority(true);
+ contexts.addHandler(lcfApi);
+ server.start();
+ }
// If all worked, then we can start the daemon.
// Clear the agents shutdown signal.
Added: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java?rev=1546865&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java Sun Dec 1 22:13:56 2013
@@ -0,0 +1,40 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** Output connector class to be used by scheduling tests. This connector never expects to see any documents at all,
+* which is fine because it rejects them all. */
+public class NullOutputConnector extends org.apache.manifoldcf.agents.output.BaseOutputConnector
+{
+
+ public NullOutputConnector()
+ {
+ super();
+ }
+
+
+}
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/NullOutputConnector.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java?rev=1546865&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java (added)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java Sun Dec 1 22:13:56 2013
@@ -0,0 +1,74 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.agents.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test of the scheduler. If the test succeeds, it is because
+* the scheduler has properly distributed requests for all bins evenly. */
+public class SchedulerHSQLDBTest extends BaseITHSQLDB
+{
+
+ protected SchedulerTester tester;
+
+ public SchedulerHSQLDBTest()
+ {
+ super(false,false);
+ tester = new SchedulerTester(mcfInstance);
+ }
+
+ @Override
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.tests.SchedulingRepositoryConnector"};
+ }
+
+ @Override
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"SchedulingConnector"};
+ }
+
+ @Override
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.tests.NullOutputConnector"};
+ }
+
+ @Override
+ protected String[] getOutputNames()
+ {
+ return new String[]{"NullOutput"};
+ }
+
+ @Test
+ public void schedulingTestRun()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+
+
+}
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java?rev=1546865&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java (added)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java Sun Dec 1 22:13:56 2013
@@ -0,0 +1,97 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a very basic sanity check */
+public class SchedulerTester
+{
+ protected ManifoldCFInstance instance;
+
+ public SchedulerTester(ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("SchedulerTest Connection");
+ conn.setDescription("SchedulerTest Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.tests.SchedulingRepositoryConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.crawler.tests.NullOutputConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("SchedulerTest Connection");
+ job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it is running.
+ jobManager.manualStart(job.getID());
+ instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+
+ // Wait for the job to become inactive. The time should be at least long enough to handle
+ // 100 documents per bin, but not significantly greater than that. Let's say 120 seconds.
+ long startTime = System.currentTimeMillis();
+ instance.waitJobInactiveNative(jobManager,job.getID(),150000L);
+ long endTime = System.currentTimeMillis();
+ if (jobManager.getStatus(job.getID()).getDocumentsProcessed() != 10+10*100)
+ throw new Exception("Expected 1010 documents, saw "+jobManager.getStatus(job.getID()).getDocumentsProcessed());
+ if (endTime-startTime < 96000L)
+ throw new Exception("Job finished too quickly; throttling clearly failed");
+ System.out.println("Crawl took "+(endTime-startTime)+" milliseconds");
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),120000L);
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java?rev=1546865&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java Sun Dec 1 22:13:56 2013
@@ -0,0 +1,143 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** Connector class to be used by scheduling tests */
+public class SchedulingRepositoryConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
+{
+ // Throttling: the next time a fetch is allowed, per bin.
+ protected static final Map<String,Long> nextFetchTime = new HashMap<String,Long>();
+
+ public SchedulingRepositoryConnector()
+ {
+ }
+
+ @Override
+ public String[] getBinNames(String documentIdentifier)
+ {
+ int index = documentIdentifier.indexOf("/");
+ return new String[]{documentIdentifier.substring(0,index)};
+ }
+
+ @Override
+ public void addSeedDocuments(ISeedingActivity activities, DocumentSpecification spec,
+ long startTime, long endTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ // A seed per domain
+ String numberDomainsString = params.getParameter("numberDomains");
+ if (numberDomainsString == null)
+ numberDomainsString = "10";
+ int numberDomains = Integer.parseInt(numberDomainsString);
+ for (int i = 0; i < numberDomains; i++)
+ {
+ activities.addSeedDocument(Integer.toString(i)+"/",null);
+ }
+ System.out.println("Seeding completed at "+System.currentTimeMillis());
+ }
+
+ @Override
+ public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
+ DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ String[] rval = new String[documentIdentifiers.length];
+ for (int i = 0; i < rval.length; i++)
+ {
+ rval[i] = "";
+ }
+ return rval;
+ }
+
+ @Override
+ public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities,
+ DocumentSpecification spec, boolean[] scanOnly, int jobMode)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ String documentsPerSeedString = params.getParameter("documentsperseed");
+ if (documentsPerSeedString == null)
+ documentsPerSeedString = "100";
+ int documentsPerSeed = Integer.parseInt(documentsPerSeedString);
+ String timePerDocumentString = params.getParameter("timeperdocument");
+ if (timePerDocumentString == null)
+ timePerDocumentString = "1000";
+ int timePerDocument = Integer.parseInt(timePerDocumentString);
+
+ // Seeds process instantly; other documents have a throttle based on the bin.
+ for (int i = 0; i < documentIdentifiers.length; i++)
+ {
+ String documentIdentifier = documentIdentifiers[i];
+ if (documentIdentifier.endsWith("/"))
+ {
+ System.out.println("Evaluating seed for "+documentIdentifier+" at "+System.currentTimeMillis());
+ // Seed document. Add the document ID's
+ for (int j = 0; j < documentsPerSeed; j++)
+ {
+ activities.addDocumentReference(documentIdentifier + Integer.toString(j),documentIdentifier,null,
+ null,null,null);
+ }
+ System.out.println("Done evaluating seed for "+documentIdentifier+" at "+System.currentTimeMillis());
+ }
+ else
+ {
+ if (!scanOnly[i])
+ {
+ System.out.println("Fetching "+documentIdentifier);
+ // Find the bin
+ String bin = documentIdentifier.substring(0,documentIdentifier.indexOf("/"));
+ // For now they are all the same
+ long binTimePerDocument = timePerDocument;
+ long now = System.currentTimeMillis();
+ long whenFetch;
+ synchronized (nextFetchTime)
+ {
+ Long time = nextFetchTime.get(bin);
+ if (time == null)
+ whenFetch = now;
+ else
+ whenFetch = time.longValue();
+ nextFetchTime.put(bin,new Long(whenFetch + binTimePerDocument));
+ }
+ if (whenFetch > now)
+ {
+ System.out.println("Waiting "+(whenFetch-now)+" to fetch "+documentIdentifier);
+ try
+ {
+ ManifoldCF.sleep(whenFetch-now);
+ }
+ catch (InterruptedException e)
+ {
+ throw new ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);
+ }
+ System.out.println("Wait complete for "+documentIdentifier);
+ }
+ }
+ }
+ }
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulingRepositoryConnector.java
------------------------------------------------------------------------------
svn:keywords = Id