You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/11/09 15:48:31 UTC
svn commit: r1540317 - in /manifoldcf/trunk: ./
framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/
framework/core/src/main/java/org/apache/manifoldcf/core/system/
framework/crawler-ui/src/main/webapp/ framework/example-multiprocess-pro...
Author: kwright
Date: Sat Nov 9 14:48:30 2013
New Revision: 1540317
URL: http://svn.apache.org/r1540317
Log:
Fix for CONNECTORS-803.
Added:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java (with props)
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml
manifoldcf/trunk/framework/example-multiprocess/properties.xml
manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml
manifoldcf/trunk/framework/example-singleprocess/properties.xml
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sat Nov 9 14:48:30 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-803: Add automatic and manual ability to clean out
+repohistory rows.
+(Marcello Lorenzi, Karl Wright)
+
CONNECTORS-801: Refactor JDBC authority to bring it into compliance with current
conventions in JDBC connector. Warning: this change is not backwards
compatible! JDBC authority queries now require explicit return column
Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/LockManagerFactory.java Sat Nov 9 14:48:30 2013
@@ -91,6 +91,12 @@ public class LockManagerFactory
{
return make(tc).getSharedConfiguration().getIntProperty(s, defaultValue);
}
+
+ public static long getLongProperty(IThreadContext tc, String s, long defaultValue)
+ throws ManifoldCFException
+ {
+ return make(tc).getSharedConfiguration().getLongProperty(s, defaultValue);
+ }
public static double getDoubleProperty(IThreadContext tc, String s, double defaultValue)
throws ManifoldCFException
Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ManifoldCFConfiguration.java Sat Nov 9 14:48:30 2013
@@ -104,6 +104,24 @@ public class ManifoldCFConfiguration ext
}
}
+ /** Read a long property, either from the system properties, or from the local configuration file.
+ */
+ public long getLongProperty(String s, long defaultValue)
+ throws ManifoldCFException
+ {
+ String value = getProperty(s);
+ if (value == null)
+ return defaultValue;
+ try
+ {
+ return Long.parseLong(value);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ManifoldCFException("Illegal property value for long property '"+s+"': '"+value+"': "+e.getMessage(),e,ManifoldCFException.SETUP_ERROR);
+ }
+ }
+
/** Read a float property, either from the system properties, or from the local configuration file.
*/
public double getDoubleProperty(String s, double defaultValue)
Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java Sat Nov 9 14:48:30 2013
@@ -393,6 +393,14 @@ public class ManifoldCF
return localConfiguration.getIntProperty(s, defaultValue);
}
+ /** Read a long property, either from the system properties, or from the local configuration file.
+ */
+ public static long getLongProperty(String s, long defaultValue)
+ throws ManifoldCFException
+ {
+ return localConfiguration.getLongProperty(s, defaultValue);
+ }
+
/** Read a float property, either from the system properties, or from the local configuration file.
*/
public static double getDoubleProperty(String s, double defaultValue)
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp Sat Nov 9 14:48:30 2013
@@ -203,6 +203,28 @@
<jsp:forward page="listconnections.jsp"/>
<%
}
+ else if (op.equals("ClearHistory"))
+ {
+ try
+ {
+ String connectionName = variableContext.getParameter("connname");
+ if (connectionName == null)
+ throw new ManifoldCFException("Missing connection parameter");
+ connManager.cleanUpHistoryData(connectionName);
+%>
+ <jsp:forward page="listconnections.jsp"/>
+<%
+ }
+ catch (ManifoldCFException e)
+ {
+ e.printStackTrace();
+ variableContext.setParameter("text",e.getMessage());
+ variableContext.setParameter("target","listconnections.jsp");
+%>
+ <jsp:forward page="error.jsp"/>
+<%
+ }
+ }
else
{
// Error
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewconnection.jsp Sat Nov 9 14:48:30 2013
@@ -45,6 +45,16 @@
}
}
+ function ClearHistory(connectionName)
+ {
+ if (confirm("<%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"viewconnection.Thiscommandwillclearallhistoryrelatedto")%> '"+connectionName+"' <%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"viewconnection.period")%>"))
+ {
+ document.viewconnection.op.value="ClearHistory";
+ document.viewconnection.connname.value=connectionName;
+ document.viewconnection.submit();
+ }
+ }
+
//-->
</script>
@@ -207,8 +217,12 @@
<tr>
<td class="separator" colspan="4"><hr/></td>
</tr>
- <tr><td class="message" colspan="4"><a href='<%="viewconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%></a> <a href='<%="editconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.EditThisConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Edit")%></a> <a href="javascript:void()" onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Deletethisconnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Delete"
)%></a>
- </td></tr>
+ <tr>
+ <td class="message" colspan="4">
+ <nobr><a href='<%="viewconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Refresh")%></a> <a href='<%="editconnection.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.EditThisConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Edit")%></a> <a href="javascript:void()" onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.Deletethisconnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.Delete")%></a></nobr>
+ <nobr><a href="javascript:void()" onclick='<%="javascript:ClearHistory(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewconnection.ClearHistoryAssociatedWithThisConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewconnection.ClearAllRelatedHistory")%></a></nobr>
+ </td>
+ </tr>
</table>
<%
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp Sat Nov 9 14:48:30 2013
@@ -159,12 +159,12 @@
<tr>
<td class="separator" colspan="4"><hr/></td>
</tr>
- <tr>
- <td class="message" colspan="4">
- <nobr><a href='<%="viewoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%></a> <a href='<%="editoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.EditThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Edit")%></a> <a href="javascript:void()" onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.DeleteThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Delete")%></a></nobr><br/>
- <nobr><a href="javascript:void()" onclick='<%="javascript:ReingestAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllAssociatedDocuments")%></a></nobr>
- </td>
- </tr>
+ <tr>
+ <td class="message" colspan="4">
+ <nobr><a href='<%="viewoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%></a> <a href='<%="editoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.EditThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Edit")%></a> <a href="javascript:void()" onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.DeleteThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Delete")%></a></nobr><br/>
+ <nobr><a href="javascript:void()" onclick='<%="javascript:ReingestAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllAssociatedDocuments")%></a></nobr>
+ </td>
+ </tr>
</table>
<%
Modified: manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml (original)
+++ manifoldcf/trunk/framework/example-multiprocess-proprietary/properties.xml Sat Nov 9 14:48:30 2013
@@ -34,6 +34,7 @@
<property name="org.apache.manifoldcf.dbsuperuserpassword" value=""/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval" value="2592000000"/>
<!-- Point to a specific (common) logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified: manifoldcf/trunk/framework/example-multiprocess/properties.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-multiprocess/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-multiprocess/properties.xml (original)
+++ manifoldcf/trunk/framework/example-multiprocess/properties.xml Sat Nov 9 14:48:30 2013
@@ -34,6 +34,7 @@
<property name="org.apache.manifoldcf.dbsuperuserpassword" value=""/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval" value="2592000000"/>
<!-- Point to a specific (common) logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified: manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml (original)
+++ manifoldcf/trunk/framework/example-singleprocess-proprietary/properties.xml Sat Nov 9 14:48:30 2013
@@ -30,6 +30,7 @@
<property name="org.apache.manifoldcf.derbydatabasepath" value="."/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval" value="2592000000"/>
<!-- Point to a specific logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified: manifoldcf/trunk/framework/example-singleprocess/properties.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/example-singleprocess/properties.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/example-singleprocess/properties.xml (original)
+++ manifoldcf/trunk/framework/example-singleprocess/properties.xml Sat Nov 9 14:48:30 2013
@@ -30,6 +30,7 @@
<property name="org.apache.manifoldcf.derbydatabasepath" value="."/>
<property name="org.apache.manifoldcf.database.maxhandles" value="100"/>
<property name="org.apache.manifoldcf.crawler.threads" value="50"/>
+ <property name="org.apache.manifoldcf.crawler.historycleanupinterval" value="2592000000"/>
<!-- Point to a specific logging file -->
<property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/>
<!-- Specify the connectors to be loaded -->
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnectionManager.java Sat Nov 9 14:48:30 2013
@@ -121,6 +121,18 @@ public interface IRepositoryConnectionMa
// Reporting and analysis related
+ /** Delete history rows related to a specific connection, upon user request.
+ *@param connectionName is the connection whose history records should be removed.
+ */
+ public void cleanUpHistoryData(String connectionName)
+ throws ManifoldCFException;
+
+ /** Delete history rows older than a specified timestamp.
+ *@param timeCutoff is the timestamp to delete older rows before.
+ */
+ public void cleanUpHistoryData(long timeCutoff)
+ throws ManifoldCFException;
+
// Activities the Connector Framework records
/** Start a job */
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java Sat Nov 9 14:48:30 2013
@@ -528,7 +528,7 @@ public class RepositoryConnectionManager
throw new ManifoldCFException("Can't delete repository connection '"+name+"': existing jobs refer to it");
ManifoldCF.noteConfigurationChange();
throttleSpecManager.deleteRows(name);
- historyManager.deleteOwner(name,null);
+ historyManager.deleteOwner(name);
ArrayList params = new ArrayList();
String query = buildConjunctionClause(params,new ClauseDescription[]{
new UnitaryClause(nameField,name)});
@@ -580,6 +580,7 @@ public class RepositoryConnectionManager
*@param className is the class name of the connector.
*@return the repository connections that use that connector.
*/
+ @Override
public String[] findConnectionsForConnector(String className)
throws ManifoldCFException
{
@@ -607,6 +608,7 @@ public class RepositoryConnectionManager
*@param name is the name of the connection to check.
*@return true if the underlying connector is registered.
*/
+ @Override
public boolean checkConnectorExists(String name)
throws ManifoldCFException
{
@@ -648,6 +650,7 @@ public class RepositoryConnectionManager
/** Return the name column.
*@return the name column.
*/
+ @Override
public String getConnectionNameColumn()
{
return nameField;
@@ -656,7 +659,26 @@ public class RepositoryConnectionManager
// Reporting and analysis related
-
+ /** Delete history rows related to a specific connection, upon user request.
+ *@param connectionName is the connection whose history records should be removed.
+ */
+ @Override
+ public void cleanUpHistoryData(String connectionName)
+ throws ManifoldCFException
+ {
+ historyManager.deleteOwner(connectionName);
+ }
+
+ /** Delete history rows older than a specified timestamp.
+ *@param timeCutoff is the timestamp to delete older rows before.
+ */
+ @Override
+ public void cleanUpHistoryData(long timeCutoff)
+ throws ManifoldCFException
+ {
+ historyManager.deleteOldRows(timeCutoff);
+ }
+
/** Record time-stamped information about the activity of the connection. This information can originate from
* either the connector or from the framework. The reason it is here is that it is viewed as 'belonging' to an
* individual connection, and is segregated accordingly.
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java Sat Nov 9 14:48:30 2013
@@ -161,15 +161,27 @@ public class RepositoryHistoryManager ex
*@param owner is the name of the owner.
*@param invKeys are the invalidation keys.
*/
- public void deleteOwner(String owner, StringSet invKeys)
+ public void deleteOwner(String owner)
throws ManifoldCFException
{
ArrayList params = new ArrayList();
String query = buildConjunctionClause(params,new ClauseDescription[]{
new UnitaryClause(ownerNameField,owner)});
- performDelete("WHERE "+query,params,invKeys);
+ performDelete("WHERE "+query,params,null);
}
+ /** Delete records older than a specified time.
+ *@param timeCutoff is the time, earlier than which records are removed.
+ */
+ public void deleteOldRows(long timeCutoff)
+ throws ManifoldCFException
+ {
+ ArrayList params = new ArrayList();
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(startTimeField,"<",new Long(timeCutoff))});
+ performDelete("WHERE "+query,params,null);
+ }
+
/** Add row to table, and reanalyze if necessary.
*/
public Long addRow(String connectionName, long startTime, long endTime, long dataSize, String activityType,
Added: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java?rev=1540317&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java Sat Nov 9 14:48:30 2013
@@ -0,0 +1,131 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.system;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import java.util.*;
+import java.lang.reflect.*;
+
+/** This class describes the thread that cleans up history records.
+* It fires infrequently and removes history records older than a configuration-determined cutoff.
+*/
+public class HistoryCleanupThread extends Thread
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ protected static final String historyCleanupIntervalProperty = "org.apache.manifoldcf.crawler.historycleanupinterval";
+
+ /** Constructor.
+ */
+ public HistoryCleanupThread()
+ throws ManifoldCFException
+ {
+ super();
+ setName("History cleanup thread");
+ setDaemon(true);
+ }
+
+ public void run()
+ {
+ try
+ {
+ // Create a thread context object.
+ IThreadContext threadContext = ThreadContextFactory.make();
+ IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
+ // Default zero value means we never clean up, which is the backwards-compatible behavior
+ long historyCleanupInterval = LockManagerFactory.getLongProperty(threadContext, historyCleanupIntervalProperty, 0L);
+ // Loop
+ while (true)
+ {
+ if (Thread.currentThread().isInterrupted())
+ break;
+
+ // Do another try/catch around everything in the loop
+ try
+ {
+ // Get current time
+ long currentTime = System.currentTimeMillis();
+ // Log it
+ if (Logging.threads.isDebugEnabled())
+ Logging.threads.debug("History cleanup thread - removing old history at "+new Long(currentTime).toString());
+ if (historyCleanupInterval > 0L && historyCleanupInterval < currentTime)
+ connectionManager.cleanUpHistoryData(currentTime - historyCleanupInterval);
+ else
+ Logging.threads.debug(" History cleanup thread did nothing because cleanup disabled");
+ // Loop around again, after resting a while
+ ManifoldCF.sleep(60L * 60L * 1000L);
+ }
+ catch (ManifoldCFException e)
+ {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ break;
+
+ if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
+ {
+ Logging.threads.error("History thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
+ try
+ {
+ // Give the database a chance to catch up/wake up
+ ManifoldCF.sleep(10000L);
+ }
+ catch (InterruptedException se)
+ {
+ break;
+ }
+ continue;
+ }
+
+ // Log it, but keep the thread alive
+ Logging.threads.error("Exception tossed: "+e.getMessage(),e);
+
+ if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR)
+ {
+ // Shut the whole system down!
+ System.exit(1);
+ }
+ }
+ catch (InterruptedException e)
+ {
+ // We're supposed to quit
+ break;
+ }
+ catch (OutOfMemoryError e)
+ {
+ System.err.println("agents process ran out of memory - shutting down");
+ e.printStackTrace(System.err);
+ System.exit(-200);
+ }
+ catch (Throwable e)
+ {
+ // A more severe error - but stay alive
+ Logging.threads.fatal("Error tossed: "+e.getMessage(),e);
+ }
+ }
+ }
+ catch (Throwable e)
+ {
+ // Severe error on initialization
+ System.err.println("agents process could not start - shutting down");
+ Logging.threads.fatal("HistoryCleanupThread initialization error tossed: "+e.getMessage(),e);
+ System.exit(-300);
+ }
+ }
+
+}
Propchange: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/HistoryCleanupThread.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Sat Nov 9 14:48:30 2013
@@ -73,6 +73,7 @@ public class ManifoldCF extends org.apac
protected static SeedingThread seedingThread = null;
protected static IdleCleanupThread idleCleanupThread = null;
protected static SetPriorityThread setPriorityThread = null;
+ protected static HistoryCleanupThread historyCleanupThread = null;
// Reset managers
/** Worker thread pool reset manager */
@@ -640,6 +641,7 @@ public class ManifoldCF extends org.apac
stufferThread = new StufferThread(documentQueue,numWorkerThreads,workerResetManager,queueTracker,blockingDocuments,lowWaterFactor,stuffAmtFactor);
expireStufferThread = new ExpireStufferThread(expireQueue,numExpireThreads,workerResetManager);
setPriorityThread = new SetPriorityThread(queueTracker,numWorkerThreads,blockingDocuments);
+ historyCleanupThread = new HistoryCleanupThread();
workerThreads = new WorkerThread[numWorkerThreads];
int i = 0;
@@ -755,6 +757,7 @@ public class ManifoldCF extends org.apac
stufferThread.start();
expireStufferThread.start();
setPriorityThread.start();
+ historyCleanupThread.start();
i = 0;
while (i < numWorkerThreads)
@@ -820,7 +823,7 @@ public class ManifoldCF extends org.apac
finisherThread != null || notificationThread != null || workerThreads != null || expireStufferThread != null || expireThreads != null ||
deleteStufferThread != null || deleteThreads != null ||
cleanupStufferThread != null || cleanupThreads != null ||
- jobResetThread != null || seedingThread != null || idleCleanupThread != null || setPriorityThread != null)
+ jobResetThread != null || seedingThread != null || idleCleanupThread != null || setPriorityThread != null || historyCleanupThread != null)
{
// Send an interrupt to all threads that are still there.
// In theory, this only needs to be done once. In practice, I have seen cases where the thread loses track of the fact that it has been
@@ -829,6 +832,10 @@ public class ManifoldCF extends org.apac
{
initializationThread.interrupt();
}
+ if (historyCleanupThread != null)
+ {
+ historyCleanupThread.interrupt();
+ }
if (setPriorityThread != null)
{
setPriorityThread.interrupt();
@@ -941,6 +948,11 @@ public class ManifoldCF extends org.apac
if (!initializationThread.isAlive())
initializationThread = null;
}
+ if (historyCleanupThread != null)
+ {
+ if (!historyCleanupThread.isAlive())
+ historyCleanupThread = null;
+ }
if (setPriorityThread != null)
{
if (!setPriorityThread.isAlive())
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Sat Nov 9 14:48:30 2013
@@ -24,6 +24,10 @@ editjob.Name=Name
editjob.Connection=Connection
viewconnection.NoneGlobalAuthority=None (global authority)
+viewconnection.ClearHistoryAssociatedWithThisConnection=Clear history associated with this connection
+viewconnection.ClearAllRelatedHistory=Clear All Related History
+viewconnection.Thiscommandwillclearallhistoryrelatedto=This command will clear all history related to connection
+viewconnection.period=.
listconnections.GlobalAuthority=None(globalAuthority)
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties Sat Nov 9 14:48:30 2013
@@ -24,6 +24,10 @@ editjob.Name=åå
editjob.Connection=ã³ãã¯ã·ã§ã³
viewconnection.NoneGlobalAuthority=ãªãï¼globalAuthorityï¼
+viewconnection.ClearHistoryAssociatedWithThisConnection=Clear history associated with this connection
+viewconnection.ClearAllRelatedHistory=Clear All Related History
+viewconnection.Thiscommandwillclearallhistoryrelatedto=This command will clear all history related to connection
+viewconnection.period=.
listconnections.GlobalAuthority=ãªãï¼globalAuthorityï¼
Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml (original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml Sat Nov 9 14:48:30 2013
@@ -836,6 +836,7 @@ cd example
<tr><td>org.apache.manifoldcf.crawler.expirethreads</td><td>No</td><td>Number of crawler expiration threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.cleanupthreads</td><td>No</td><td>Number of crawler cleanup threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.deletethreads</td><td>No</td><td>Number of crawler delete threads created. Suggest a value of 10.</td></tr>
+ <tr><td>org.apache.manifoldcf.crawler.historycleanupinterval</td><td>No</td><td>Milliseconds to retain history records. Default is 0. Zero means "forever".</td></tr>
<tr><td>org.apache.manifoldcf.misc</td><td>No</td><td>Miscellaneous debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.db</td><td>No</td><td>Database debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.lock</td><td>No</td><td>Lock management debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml?rev=1540317&r1=1540316&r2=1540317&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml (original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml Sat Nov 9 14:48:30 2013
@@ -766,6 +766,8 @@ cd example
when operating without any apparent stalls due to the above issues, Derby is still only about 1/4 as fast as PostgreSQL. At the moment this limits Derby's utility for
ManifoldCF to demonstration and testing.</p>
</section>
+
+
</section>
<section>
@@ -834,6 +836,7 @@ cd example
<tr><td>org.apache.manifoldcf.crawler.expirethreads</td><td>No</td><td>Number of crawler expiration threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.cleanupthreads</td><td>No</td><td>Number of crawler cleanup threads created. Suggest a value of 10.</td></tr>
<tr><td>org.apache.manifoldcf.crawler.deletethreads</td><td>No</td><td>Number of crawler delete threads created. Suggest a value of 10.</td></tr>
+ <tr><td>org.apache.manifoldcf.crawler.historycleanupinterval</td><td>No</td><td>Milliseconds to retain history records. Default is 0. Zero means "forever".</td></tr>
<tr><td>org.apache.manifoldcf.misc</td><td>No</td><td>Miscellaneous debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.db</td><td>No</td><td>Database debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
<tr><td>org.apache.manifoldcf.lock</td><td>No</td><td>Lock management debugging output. Legal values INFO, WARN, or DEBUG.</td></tr>
@@ -940,7 +943,47 @@ mvn exec:exec
<p>In a multi process setup, all of the ManifoldCF processes might as well exist on their own. You can learn how to programmatically start the agents process by looking at the code
in the AgentRun command class, as described above. Similarly, the command classes that register connectors are very small and should be easy to understand.</p>
</section>
-
+
+ <section>
+ <title>Integrating ManifoldCF with a search engine</title>
+ <p></p>
+ <p>ManifoldCF's Authority Service is designed to allow maximum flexibility in integrating ManifoldCF security with search engines. The
+ service receives a user identity (as a set of authorization domain/user name tuples), and produces a set of tokens. It also returns a
+ summary of the status of all authorities that were involved in the assembly of the set of tokens, as a nicety. A search engine user
+ interface could thus signal the user when the results they might be seeing are incomplete, and why.</p>
+ <p>The Authority Service expects the following arguments, passed as URL arguments and properly URL encoded:</p>
+ <p></p>
+ <table>
+ <caption>Authority Service URL parameters</caption>
+ <tr><th>Authority Service URL parameter</th><th>Meaning</th></tr>
+ <tr><td>username</td><td>the username, if there is only one authorization domain</td></tr>
+ <tr><td>domain</td><td>the optional authorization domain if there is only one authorization domain (defaults to empty string)</td></tr>
+ <tr><td>username_<em>XX</em></td><td>username number <em>XX</em>, where <em>XX</em> is an integer starting at zero</td></tr>
+ <tr><td>domain_<em>XX</em></td><td>authorization domain <em>XX</em>, where <em>XX</em> is an integer starting at zero</td></tr>
+ </table>
+ <p></p>
+ <p>Access tokens and authority statuses are returned in the HTTP response separated by newline characters. Each line has a prefix
+ as follows:</p>
+ <p></p>
+ <table>
+ <caption>Authority Service response prefixes</caption>
+ <tr><th>Authority Service response prefix</th><th>Meaning</th></tr>
+ <tr><td>TOKEN:</td><td>An access token</td></tr>
+ <tr><td>AUTHORIZED:</td><td>The name of an authority that found the user to be authorized</td></tr>
+ <tr><td>UNREACHABLEAUTHORITY:</td><td>The name of an authority that was found to be unreachable or unusable</td></tr>
+ <tr><td>UNAUTHORIZED:</td><td>The name of an authority that found the user to be unauthorized</td></tr>
+ <tr><td>USERNOTFOUND:</td><td>The name of an authority that could not find the user</td></tr>
+ </table>
+ <p></p>
+ <p>It is important to remember that only the "TOKEN:" lines actually matter for security. Even if any of the error conditions apply, the set
+ of tokens returned by the Authority Service will be correctly supplied in order to apply appropriate security to documents being searched.</p>
+ <p>If you choose to deploy a search-engine plugin supplied by the Apache ManifoldCF project (for example, the Solr plugin), you will not need
+ know any of the above, since part of the plugin's purpose is to communicate with the Authority Service and apply the access tokens that are
+ returned to the search query automatically. Some plugins, such as the ElasticSearch plugin, are more or less like toolkits, but still hide most
+ of the above from the integrator. In a more highly customized system, however, you may need to develop your own code which interacts
+ with the Authority Service in order to meet your goals.</p>
+ </section>
+
</section>
</body>