You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/10 14:27:07 UTC

svn commit: r1549837 - in /manifoldcf/trunk: ./ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/ framework/agents/src/main/java/org/apache/manifoldcf/...

Author: kwright
Date: Tue Dec 10 13:27:06 2013
New Revision: 1549837

URL: http://svn.apache.org/r1549837
Log:
Fix for CONNECTORS-827.  Add button on output connection view page that allows ingeststatus records to be removed entirely, in case the connection is permanently dead rather than just reconfigured.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
    manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
    manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
    manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
    manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Tue Dec 10 13:27:06 2013
@@ -3,6 +3,11 @@ $Id$
 
 ======================= 1.5-dev =====================
 
+CONNECTORS-827: Add a feature which removes all ingeststatus
+records for an output connection, in case that the target index
+is gone forever and is inaccessible.
+(Swami Rajamohan, Karl Wright)
+
 CONNECTORS-795, CONNECTORS-821: Release new versions of
 Solr 3.x, Solr 4.x, ElasticSearch, and SharePoint 2010 plugins.
 (Karl Wright)

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Tue Dec 10 13:27:06 2013
@@ -1388,6 +1388,21 @@ public class IncrementalIngester extends
     performUpdate(map,"WHERE "+query,list,null);
   }
 
+  /** Remove all knowledge of an output index from the system.  This is appropriate
+  * when the output index no longer exists and you wish to delete the associated job.
+  *@param outputConnectionName is the name of the output connection associated with this action.
+  */
+  @Override
+  public void removeOutputConnection(String outputConnectionName)
+    throws ManifoldCFException
+  {
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
+    performDelete("WHERE "+query,list,null);
+  }
+  
   /** Note the ingestion of a document, or the "update" of a document.
   *@param outputConnectionName is the name of the output connection.
   *@param docKey is the key string describing the document.

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java Tue Dec 10 13:27:06 2013
@@ -297,4 +297,11 @@ public interface IIncrementalIngester
   public void resetOutputConnection(String outputConnectionName)
     throws ManifoldCFException;
     
+  /** Remove all knowledge of an output index from the system.  This is appropriate
+  * when the output index no longer exists and you wish to delete the associated job.
+  *@param outputConnectionName is the name of the output connection associated with this action.
+  */
+  public void removeOutputConnection(String outputConnectionName)
+    throws ManifoldCFException;
+
 }

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java Tue Dec 10 13:27:06 2013
@@ -145,6 +145,23 @@ public class ManifoldCF extends org.apac
     // resulting from this signal that find themselves "unchanged".
     AgentManagerFactory.noteOutputConnectionChange(threadContext,connectionName);
   }
+
+  /** Signal output connection has been deleted.
+  * This is called when the target of an output connection has been removed,
+  * therefore all associated documents were also already removed.
+  *@param threadContext is the thread context.
+  *@param connectionName is the connection name.
+  */
+  public static void signalOutputConnectionRemoved(IThreadContext threadContext, String connectionName)
+    throws ManifoldCFException
+  {
+    // Blow away the incremental ingestion table first
+    IIncrementalIngester ingester = IncrementalIngesterFactory.make(threadContext);
+    ingester.removeOutputConnection(connectionName);
+    // Now, signal to all agents that the output connection configuration has changed.  Do this second, so that there cannot be documents
+    // resulting from this signal that find themselves "unchanged".
+    AgentManagerFactory.noteOutputConnectionChange(threadContext,connectionName);
+  }
   
   // Helper methods for API support.  These are made public so connectors can use them to implement the executeCommand method.
   

Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp Tue Dec 10 13:27:06 2013
@@ -716,6 +716,28 @@
 <%
 				}
 			}
+			else if (op.equals("RemoveAll"))
+			{
+				try
+				{
+					String connectionName = variableContext.getParameter("connname");
+					if (connectionName == null)
+						throw new ManifoldCFException("Missing connection parameter");
+					org.apache.manifoldcf.agents.system.ManifoldCF.signalOutputConnectionRemoved(threadContext,connectionName);
+%>
+					<jsp:forward page="listoutputs.jsp"/>
+<%
+				}
+				catch (ManifoldCFException e)
+				{
+					e.printStackTrace();
+					variableContext.setParameter("text",e.getMessage());
+					variableContext.setParameter("target","listoutputs.jsp");
+%>
+					<jsp:forward page="error.jsp"/>
+<%
+				}
+			}
 			else
 			{
 				// Error

Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewoutput.jsp Tue Dec 10 13:27:06 2013
@@ -55,6 +55,16 @@
 		}
 	}
 	
+	function RemoveAll(connectionName)
+	{
+		if (confirm("<%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"viewoutput.Thiscommandwillcause")%> '"+connectionName+"' <%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"viewoutput.tobeforgotten")%>"))
+		{
+			document.viewconnection.op.value="RemoveAll";
+			document.viewconnection.connname.value=connectionName;
+			document.viewconnection.submit();
+		}
+	}
+
 	//-->
 	</script>
 
@@ -163,7 +173,7 @@
 			<tr>
 				<td class="message" colspan="4">
 					<nobr><a href='<%="viewoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Refresh")%></a>&nbsp;<a href='<%="editoutput.jsp?connname="+java.net.URLEncoder.encode(connectionName,"UTF-8")%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.EditThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Edit")%></a>&nbsp;<a href="javascript:void()" onclick='<%="javascript:Delete(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.DeleteThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.Delete")%></a></nobr><br/>
-					<nobr><a href="javascript:void()" onclick='<%="javascript:ReingestAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllAssociatedDocuments")%></a></nobr>
+					<nobr><a href="javascript:void()" onclick='<%="javascript:ReingestAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.ReIngestAllAssociatedDocuments")%></a>&nbsp;<a href="javascript:void()" onclick='<%="javascript:RemoveAll(\""+org.apache.manifoldcf.ui.util.Encoder.attributeJavascriptEscape(connectionName)+"\")"%>' alt="<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"viewoutput.RemoveAllDocumentsAssociatedWithThisOutputConnection")%>"><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewoutput.RemoveAllAssociatedDocuments")%></a></nobr>
 				</td>
 			</tr>
 		</table>

Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Tue Dec 10 13:27:06 2013
@@ -132,9 +132,13 @@ viewoutput.DeleteThisOutputConnection=De
 viewoutput.Delete=Delete
 viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection=Re-ingest all documents associated with this output connection
 viewoutput.ReIngestAllAssociatedDocuments=Re-ingest all associated documents
+viewoutput.RemoveAllDocumentsAssociatedWithThisOutputConnection=Remove all document records associated with this output connection
+viewoutput.RemoveAllAssociatedDocuments=Remove all associated documents
 viewoutput.Deleteoutputconnection=Delete output connection
 viewoutput.Thiscommandwillforce=This command will force all documents associated with output\nconnection
 viewoutput.toberecrawled=to be recrawled the next time their associated\n jobs are started.  Do you want to continue?
+viewoutput.Thiscommandwillcause=This command will cause ManifoldCF to lose all current knowledge of documents\n associated with output connection
+viewoutput.tobeforgotten=.  Do you want to continue?
 viewoutput.qmark=?
 viewoutput.uninstalled=(uninstalled)
 viewoutput.Connectorisnotinstalled=Connector is not installed.

Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties?rev=1549837&r1=1549836&r2=1549837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties Tue Dec 10 13:27:06 2013
@@ -132,9 +132,13 @@ viewoutput.DeleteThisOutputConnection=å
 viewoutput.Delete=削除
 viewoutput.ReIngestAllDocumentsAssociatedWithThisOutputConnection=コネクションに指定されているすべてのコンテンツを再読込む
 viewoutput.ReIngestAllAssociatedDocuments=すべてのコンテンツの再読込み
+viewoutput.RemoveAllDocumentsAssociatedWithThisOutputConnection=Remove all document records associated with this output connection
+viewoutput.RemoveAllAssociatedDocuments=Remove all associated documents
 viewoutput.Deleteoutputconnection=出力コネクションを削除
 viewoutput.Thiscommandwillforce=このコマンドは、出力コネクションに関連付けられているすべてのドキュメントに強制されます
 viewoutput.toberecrawled=次回それらの関連するジョブが開始する時に再クロールされます。  継続しますか？
+viewoutput.Thiscommandwillcause=This command will cause ManifoldCF to lose all current knowledge of documents\n associated with output connection
+viewoutput.tobeforgotten=.  Do you want to continue?
 viewoutput.qmark=？
 viewoutput.uninstalled=（アンインストール）
 viewoutput.Connectorisnotinstalled=コネクターがインストールされていません