You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/08/15 12:24:29 UTC
svn commit: r1618145 - in /manifoldcf/branches/release-1.7-branch: ./
CHANGES.txt
framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Author: kwright
Date: Fri Aug 15 10:24:29 2014
New Revision: 1618145
URL: http://svn.apache.org/r1618145
Log:
Pull up fix for CONNECTORS-1010 from trunk
Modified:
manifoldcf/branches/release-1.7-branch/ (props changed)
manifoldcf/branches/release-1.7-branch/CHANGES.txt
manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Propchange: manifoldcf/branches/release-1.7-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1618144
Modified: manifoldcf/branches/release-1.7-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/CHANGES.txt?rev=1618145&r1=1618144&r2=1618145&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.7-branch/CHANGES.txt Fri Aug 15 10:24:29 2014
@@ -3,6 +3,10 @@ $Id$
======================= Release 1.7 =====================
+CONNECTORS-1010: Use uri hash instead of full URI as lock key
+to prevent "file name too long" errors on windows.
+(Erlend Garåsen, Shigeki Kobayashi, Karl Wright)
+
CONNECTORS-1007: Fix CMIS connector so that tests pass.
(Karl Wright)
Modified: manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1618145&r1=1618144&r2=1618145&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Fri Aug 15 10:24:29 2014
@@ -1036,13 +1036,15 @@ public class IncrementalIngester extends
}
String[] lockArray = new String[validURIcount];
String[] validURIArray = new String[validURIcount];
+ String[] validURIHashArray = new String[validURIcount];
validURIcount = 0;
for (int i = 0; i < uris.length; i++)
{
if (uris[i] != null && uris[i].getURI() != null)
{
validURIArray[validURIcount] = uris[i].getURI();
- lockArray[validURIcount] = outputConnectionName+":"+validURIArray[validURIcount];
+ validURIHashArray[validURIcount] = uris[i].getURIHash();
+ lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
validURIcount++;
}
}
@@ -1079,9 +1081,11 @@ public class IncrementalIngester extends
Set<String> docURIValues = new HashSet<String>();
for (String docDBString : validURIArray)
{
- String docDBHashString = ManifoldCF.hash(docDBString);
docURIValues.add(docDBString);
- docURIHashValues.add(docDBHashString);
+ }
+ for (String docDBString : validURIHashArray)
+ {
+ docURIHashValues.add(docDBString);
}
// Now, perform n queries, each of them no larger the maxInClause in length.
@@ -1248,13 +1252,15 @@ public class IncrementalIngester extends
}
String[] lockArray = new String[validURIcount];
String[] validURIArray = new String[validURIcount];
+ String[] validURIHashArray = new String[validURIcount];
validURIcount = 0;
for (int i = 0; i < uris.length; i++)
{
if (uris[i] != null && uris[i].getURI() != null)
{
validURIArray[validURIcount] = uris[i].getURI();
- lockArray[validURIcount] = outputConnectionName+":"+validURIArray[validURIcount];
+ validURIHashArray[validURIcount] = uris[i].getURIHash();
+ lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
validURIcount++;
}
}
@@ -1291,9 +1297,11 @@ public class IncrementalIngester extends
Set<String> docURIValues = new HashSet<String>();
for (String docDBString : validURIArray)
{
- String docDBHashString = ManifoldCF.hash(docDBString);
docURIValues.add(docDBString);
- docURIHashValues.add(docDBHashString);
+ }
+ for (String docDBString : validURIHashArray)
+ {
+ docURIHashValues.add(docDBString);
}
// Now, perform n queries, each of them no larger the maxInClause in length.
@@ -2237,7 +2245,7 @@ public class IncrementalIngester extends
new MultiClause(docKeyField,list),
new UnitaryClause(outputConnNameField,outputConnectionName)});
- IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
+ IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
query,newList,null,null);
// Go through list and put into buckets.
@@ -2245,14 +2253,17 @@ public class IncrementalIngester extends
{
IResultRow row = set.getRow(i);
String docHash = row.getValue(docKeyField).toString();
- Integer position = (Integer)map.get(docHash);
+ Integer position = map.get(docHash);
if (position != null)
{
String lastURI = (String)row.getValue(docURIField);
if (lastURI != null && lastURI.length() == 0)
lastURI = null;
+ String lastURIHash = (String)row.getValue(uriHashField);
+ if (lastURIHash != null && lastURIHash.length() == 0)
+ lastURIHash = null;
String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
- rval[position.intValue()] = new DeleteInfo(lastURI,lastOutputVersion);
+ rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
}
}
}
@@ -2283,7 +2294,7 @@ public class IncrementalIngester extends
new UnitaryClause(outputConnNameField,outputConnectionName),
((componentHash==null)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash))});
- IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
+ IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
query,newList,null,null);
// Go through list and put into buckets.
@@ -2297,8 +2308,11 @@ public class IncrementalIngester extends
String lastURI = (String)row.getValue(docURIField);
if (lastURI != null && lastURI.length() == 0)
lastURI = null;
+ String lastURIHash = (String)row.getValue(uriHashField);
+ if (lastURIHash != null && lastURIHash.length() == 0)
+ lastURIHash = null;
String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
- rval[position.intValue()] = new DeleteInfo(lastURI,lastOutputVersion);
+ rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
}
}
}
@@ -2351,11 +2365,13 @@ public class IncrementalIngester extends
protected static class DeleteInfo
{
protected String uriValue;
+ protected String uriHashValue;
protected String outputVersion;
- public DeleteInfo(String uriValue, String outputVersion)
+ public DeleteInfo(String uriValue, String uriHashValue, String outputVersion)
{
this.uriValue = uriValue;
+ this.uriHashValue = uriHashValue;
this.outputVersion = outputVersion;
}
@@ -2364,6 +2380,11 @@ public class IncrementalIngester extends
return uriValue;
}
+ public String getURIHash()
+ {
+ return uriHashValue;
+ }
+
public String getOutputVersion()
{
return outputVersion;
@@ -3325,7 +3346,7 @@ public class IncrementalIngester extends
// But, since we need to insure that any given URI is only worked on by one thread at a time, use critical sections
// to block the rare case that multiple threads try to work on the same URI.
- String[] lockArray = computeLockArray(documentURI,oldURI,outputConnectionName);
+ String[] lockArray = computeLockArray(documentURIHash,oldURIHash,outputConnectionName);
lockManager.enterLocks(null,null,lockArray);
try
{