You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/21 16:42:36 UTC
svn commit: r1640944 - in /manifoldcf/branches/dev_1x: ./ CHANGES.txt
framework/
framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Author: kwright
Date: Fri Nov 21 15:42:36 2014
New Revision: 1640944
URL: http://svn.apache.org/r1640944
Log:
Pull up fix for CONNECTORS-1111 from trunk.
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/CHANGES.txt
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1640941-1640942
Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1640944&r1=1640943&r2=1640944&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Fri Nov 21 15:42:36 2014
@@ -3,6 +3,9 @@ $Id$
======================= 1.8-dev =====================
+CONNECTORS-1111: Component cleanup did not work properly.
+(Markus Schuch, Karl Wright)
+
CONNECTORS-1110: Component documents getting deleted wrongly
when additional components added.
(Markus Schuch, Karl Wright)
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/framework:r1640941
Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1640944&r1=1640943&r2=1640944&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Fri Nov 21 15:42:36 2014
@@ -1052,26 +1052,26 @@ public class IncrementalIngester extends
// No transactions. Time for the operation may exceed transaction timeout.
// Obtain the current URIs of all of these.
- DeleteInfo[] uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes);
+ List<DeleteInfo> uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes);
// Grab critical section locks so that we can't attempt to ingest at the same time we are deleting.
// (This guarantees that when this operation is complete the database reflects reality.)
int validURIcount = 0;
- for (int i = 0; i < uris.length; i++)
+ for (DeleteInfo uri : uris)
{
- if (uris[i] != null && uris[i].getURI() != null)
+ if (uri.getURI() != null)
validURIcount++;
}
String[] lockArray = new String[validURIcount];
String[] validURIArray = new String[validURIcount];
String[] validURIHashArray = new String[validURIcount];
validURIcount = 0;
- for (int i = 0; i < uris.length; i++)
+ for (DeleteInfo uri : uris)
{
- if (uris[i] != null && uris[i].getURI() != null)
+ if (uri.getURI() != null)
{
- validURIArray[validURIcount] = uris[i].getURI();
- validURIHashArray[validURIcount] = uris[i].getURIHash();
+ validURIArray[validURIcount] = uri.getURI();
+ validURIHashArray[validURIcount] = uri.getURIHash();
lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
validURIcount++;
}
@@ -1081,10 +1081,10 @@ public class IncrementalIngester extends
try
{
// Fetch the document URIs for the listed documents
- for (int i = 0; i < uris.length; i++)
+ for (DeleteInfo uri : uris)
{
- if (uris[i] != null && uris[i].getURI() != null)
- removeDocument(connection,uris[i].getURI(),uris[i].getOutputVersion(),activities);
+ if (uri.getURI() != null)
+ removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities);
}
// Now, get rid of all rows that match the given uris.
@@ -1268,26 +1268,26 @@ public class IncrementalIngester extends
// No transactions. Time for the operation may exceed transaction timeout.
// Obtain the current URIs of all of these.
- DeleteInfo[] uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash);
+ List<DeleteInfo> uris = getDocumentURIMultiple(outputConnectionName,identifierClasses,identifierHashes,componentHash);
// Grab critical section locks so that we can't attempt to ingest at the same time we are deleting.
// (This guarantees that when this operation is complete the database reflects reality.)
int validURIcount = 0;
- for (int i = 0; i < uris.length; i++)
+ for (DeleteInfo uri : uris)
{
- if (uris[i] != null && uris[i].getURI() != null)
+ if (uri.getURI() != null)
validURIcount++;
}
String[] lockArray = new String[validURIcount];
String[] validURIArray = new String[validURIcount];
String[] validURIHashArray = new String[validURIcount];
validURIcount = 0;
- for (int i = 0; i < uris.length; i++)
+ for (DeleteInfo uri : uris)
{
- if (uris[i] != null && uris[i].getURI() != null)
+ if (uri.getURI() != null)
{
- validURIArray[validURIcount] = uris[i].getURI();
- validURIHashArray[validURIcount] = uris[i].getURIHash();
+ validURIArray[validURIcount] = uri.getURI();
+ validURIHashArray[validURIcount] = uri.getURIHash();
lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
validURIcount++;
}
@@ -1297,10 +1297,10 @@ public class IncrementalIngester extends
try
{
// Fetch the document URIs for the listed documents
- for (int i = 0; i < uris.length; i++)
+ for (DeleteInfo uri : uris)
{
- if (uris[i] != null && uris[i].getURI() != null)
- removeDocument(connection,uris[i].getURI(),uris[i].getOutputVersion(),activities);
+ if (uri.getURI() != null)
+ removeDocument(connection,uri.getURI(),uri.getOutputVersion(),activities);
}
// Now, get rid of all rows that match the given uris.
@@ -1615,37 +1615,29 @@ public class IncrementalIngester extends
*@return the array of current document uri's. Null returned for identifiers
* that don't exist in the index.
*/
- protected DeleteInfo[] getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes)
+ protected List<DeleteInfo> getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes)
throws ManifoldCFException
{
- DeleteInfo[] rval = new DeleteInfo[identifierHashes.length];
- Map<String,Integer> map = new HashMap<String,Integer>();
- for (int i = 0; i < identifierHashes.length; i++)
- {
- map.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i));
- rval[i] = null;
- }
-
+ List<DeleteInfo> rval = new ArrayList<DeleteInfo>();
beginTransaction();
try
{
List<String> list = new ArrayList<String>();
int maxCount = maxClauseDocumentURIChunk(outputConnectionName);
int j = 0;
- Iterator<String> iter = map.keySet().iterator();
- while (iter.hasNext())
+ for (int i = 0; i < identifierHashes.length; i++)
{
if (j == maxCount)
{
- getDocumentURIChunk(rval,map,outputConnectionName,list);
+ getDocumentURIChunk(rval,outputConnectionName,list);
j = 0;
list.clear();
}
- list.add(iter.next());
+ list.add(makeKey(identifierClasses[i],identifierHashes[i]));
j++;
}
if (j > 0)
- getDocumentURIChunk(rval,map,outputConnectionName,list);
+ getDocumentURIChunk(rval,outputConnectionName,list);
return rval;
}
catch (ManifoldCFException e)
@@ -1672,37 +1664,29 @@ public class IncrementalIngester extends
*@return the array of current document uri's. Null returned for identifiers
* that don't exist in the index.
*/
- protected DeleteInfo[] getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes, String componentHash)
+ protected List<DeleteInfo> getDocumentURIMultiple(String outputConnectionName, String[] identifierClasses, String[] identifierHashes, String componentHash)
throws ManifoldCFException
{
- DeleteInfo[] rval = new DeleteInfo[identifierHashes.length];
- Map<String,Integer> map = new HashMap<String,Integer>();
- for (int i = 0; i < identifierHashes.length; i++)
- {
- map.put(makeKey(identifierClasses[i],identifierHashes[i]),new Integer(i));
- rval[i] = null;
- }
-
+ List<DeleteInfo> rval = new ArrayList<DeleteInfo>();
beginTransaction();
try
{
List<String> list = new ArrayList<String>();
int maxCount = maxClauseDocumentURIChunk(outputConnectionName,componentHash);
int j = 0;
- Iterator<String> iter = map.keySet().iterator();
- while (iter.hasNext())
+ for (int i = 0; i < identifierHashes.length; i++)
{
if (j == maxCount)
{
- getDocumentURIChunk(rval,map,outputConnectionName,list,componentHash);
+ getDocumentURIChunk(rval,outputConnectionName,list,componentHash);
j = 0;
list.clear();
}
- list.add(iter.next());
+ list.add(makeKey(identifierClasses[i],identifierHashes[i]));
j++;
}
if (j > 0)
- getDocumentURIChunk(rval,map,outputConnectionName,list,componentHash);
+ getDocumentURIChunk(rval,outputConnectionName,list,componentHash);
return rval;
}
catch (ManifoldCFException e)
@@ -2264,7 +2248,7 @@ public class IncrementalIngester extends
*@param clause is the in clause for the query.
*@param list are the doc keys for the query.
*/
- protected void getDocumentURIChunk(DeleteInfo[] rval, Map<String,Integer> map, String outputConnectionName,
+ protected void getDocumentURIChunk(List<DeleteInfo> rval, String outputConnectionName,
List<String> list)
throws ManifoldCFException
{
@@ -2280,19 +2264,15 @@ public class IncrementalIngester extends
for (int i = 0; i < set.getRowCount(); i++)
{
IResultRow row = set.getRow(i);
- String docHash = row.getValue(docKeyField).toString();
- Integer position = map.get(docHash);
- if (position != null)
- {
- String lastURI = (String)row.getValue(docURIField);
- if (lastURI != null && lastURI.length() == 0)
- lastURI = null;
- String lastURIHash = (String)row.getValue(uriHashField);
- if (lastURIHash != null && lastURIHash.length() == 0)
- lastURIHash = null;
- String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
- rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
- }
+ //String docHash = row.getValue(docKeyField).toString();
+ String lastURI = (String)row.getValue(docURIField);
+ if (lastURI != null && lastURI.length() == 0)
+ lastURI = null;
+ String lastURIHash = (String)row.getValue(uriHashField);
+ if (lastURIHash != null && lastURIHash.length() == 0)
+ lastURIHash = null;
+ String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
+ rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion));
}
}
@@ -2312,7 +2292,7 @@ public class IncrementalIngester extends
*@param list are the doc keys for the query.
*@param componentHash is the component hash, if any, for the query.
*/
- protected void getDocumentURIChunk(DeleteInfo[] rval, Map<String,Integer> map, String outputConnectionName,
+ protected void getDocumentURIChunk(List<DeleteInfo> rval, String outputConnectionName,
List<String> list, String componentHash)
throws ManifoldCFException
{
@@ -2329,19 +2309,15 @@ public class IncrementalIngester extends
for (int i = 0; i < set.getRowCount(); i++)
{
IResultRow row = set.getRow(i);
- String docHash = row.getValue(docKeyField).toString();
- Integer position = (Integer)map.get(docHash);
- if (position != null)
- {
- String lastURI = (String)row.getValue(docURIField);
- if (lastURI != null && lastURI.length() == 0)
- lastURI = null;
- String lastURIHash = (String)row.getValue(uriHashField);
- if (lastURIHash != null && lastURIHash.length() == 0)
- lastURIHash = null;
- String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
- rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
- }
+ //String docHash = row.getValue(docKeyField).toString();
+ String lastURI = (String)row.getValue(docURIField);
+ if (lastURI != null && lastURI.length() == 0)
+ lastURI = null;
+ String lastURIHash = (String)row.getValue(uriHashField);
+ if (lastURIHash != null && lastURIHash.length() == 0)
+ lastURIHash = null;
+ String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
+ rval.add(new DeleteInfo(lastURI,lastURIHash,lastOutputVersion));
}
}