You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/08/08 20:56:20 UTC
svn commit: r1370892 - in /manifoldcf/branches/CONNECTORS-501:
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
Author: kwright
Date: Wed Aug 8 18:56:20 2012
New Revision: 1370892
URL: http://svn.apache.org/viewvc?rev=1370892&view=rev
Log:
Revert the critical parts of CONNECTORS-464 in order to get the hopcount cutoff stuff working correctly again.
Modified:
manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
Modified: manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java?rev=1370892&r1=1370891&r2=1370892&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java (original)
+++ manifoldcf/branches/CONNECTORS-501/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java Wed Aug 8 18:56:20 2012
@@ -591,13 +591,11 @@ public class HopCount extends org.apache
//
// ... and then, re-evaluate all hopcount records and their dependencies that are marked for delete.
//
- // But, the trick is that both source and target links must go away!! So deleting a document is very different than
- // updating a link...
+
// This also removes the links themselves...
if (hopcountMethod == IJobDescription.HOPCOUNT_ACCURATE)
doDeleteDocuments(jobID,documentHashes);
-
}
catch (ManifoldCFException e)
@@ -1084,7 +1082,7 @@ public class HopCount extends org.apache
}
- /** Invalidate links that start with or end in a specific set of documents, described by
+ /** Invalidate links that start with a specific set of documents, described by
* a table join.
*/
protected void doDeleteDocuments(Long jobID,
@@ -1141,39 +1139,7 @@ public class HopCount extends org.apache
performUpdate(map,sb.toString(),list,null);
noteModifications(0,1,0);
-
- sb = new StringBuilder("WHERE ");
- list = new ArrayList();
-
- sb.append(idField).append(" IN(SELECT t0.").append(deleteDepsManager.ownerIDField).append(" FROM ")
- .append(deleteDepsManager.getTableName()).append(" t0,").append(joinTableName).append(",")
- .append(intrinsicLinkManager.getTableName()).append(" t1 WHERE ");
-
- sb.append(buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause("t0."+deleteDepsManager.jobIDField,jobID)})).append(" AND ");
-
- sb.append(buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause("t1."+intrinsicLinkManager.jobIDField,jobID),
- new JoinClause("t1."+intrinsicLinkManager.parentIDHashField,"t0."+deleteDepsManager.parentIDHashField),
- new JoinClause("t1."+intrinsicLinkManager.linkTypeField,"t0."+deleteDepsManager.linkTypeField),
- new JoinClause("t1."+intrinsicLinkManager.childIDHashField,"t0."+deleteDepsManager.childIDHashField)})).append(" AND ");
-
- sb.append(buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause(joinTableJobColumn,jobID),
- new JoinClause(joinTableIDColumn,"t0."+deleteDepsManager.parentIDHashField)})).append(" AND ");
-
- sb.append(joinTableCriteria);
- list.addAll(joinTableParams);
-
- sb.append(")");
-
- map = new HashMap();
- // These are whacked back to "infinity" to avoid infinite looping in a cut-off graph.
- map.put(distanceField,new Long(-1L));
- map.put(markForDeathField,markToString(MARK_DELETING));
- performUpdate(map,sb.toString(),list,null);
- noteModifications(0,1,0);
-
+ // We do NOT do the parentID because otherwise we have the potential to delete links that we need later. See CONNECTORS-501.
if (Logging.hopcount.isDebugEnabled())
Logging.hopcount.debug("Done setting hopcount rows for job "+jobID+" to initial distances");
@@ -1206,7 +1172,7 @@ public class HopCount extends org.apache
}
- /** Invalidate links that start with or end in a specific set of documents.
+ /** Invalidate links that start with a specific set of documents.
*/
protected void doDeleteDocuments(Long jobID,
String[] documentHashes)
@@ -1327,28 +1293,8 @@ public class HopCount extends org.apache
map.put(markForDeathField,markToString(MARK_DELETING));
performUpdate(map,sb.toString(),thisList,null);
- sb = new StringBuilder("WHERE ");
- thisList = new ArrayList();
-
- sb.append(idField).append(" IN(SELECT ").append(deleteDepsManager.ownerIDField).append(" FROM ").append(deleteDepsManager.getTableName()).append(" t0 WHERE ")
- .append(buildConjunctionClause(thisList,new ClauseDescription[]{
- new UnitaryClause("t0."+deleteDepsManager.jobIDField,jobID),
- new MultiClause("t0."+deleteDepsManager.parentIDHashField,list)})).append(" AND ");
-
- sb.append("EXISTS(SELECT 'x' FROM ").append(intrinsicLinkManager.getTableName()).append(" t1 WHERE ")
- .append(buildConjunctionClause(thisList,new ClauseDescription[]{
- new JoinClause("t1."+intrinsicLinkManager.jobIDField,"t0."+deleteDepsManager.jobIDField),
- new JoinClause("t1."+intrinsicLinkManager.linkTypeField,"t0."+deleteDepsManager.linkTypeField),
- new JoinClause("t1."+intrinsicLinkManager.parentIDHashField,"t0."+deleteDepsManager.parentIDHashField),
- new JoinClause("t1."+intrinsicLinkManager.childIDHashField,"t0."+deleteDepsManager.childIDHashField)}));
-
- sb.append("))");
-
- map = new HashMap();
- // These are whacked back to "infinity" to avoid infinite looping in a cut-off graph.
- map.put(distanceField,new Long(-1L));
- map.put(markForDeathField,markToString(MARK_DELETING));
- performUpdate(map,sb.toString(),thisList,null);
+ // We do NOT do the parentID because we need to leave intrinsic links around that could be used again.
+ // See CONNECTORS-501.
}
/** Invalidate links meeting a simple criteria which have a given set of source documents. This also runs a queue
Modified: manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java?rev=1370892&r1=1370891&r2=1370892&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java (original)
+++ manifoldcf/branches/CONNECTORS-501/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java Wed Aug 8 18:56:20 2012
@@ -134,7 +134,7 @@ public class HopcountTester
// But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
-
+
// Now, delete the job.
jobManager.deleteJob(job.getID());
instance.waitJobDeletedNative(jobManager,job.getID(), 120000L);