You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2010/03/02 16:32:36 UTC
svn commit: r918079 - in /incubator/lcf/trunk/modules:
connectors/livelink/crawler-ui/connectors/livelink/
connectors/meridio/crawler-ui/authorities/meridio/
connectors/meridio/crawler-ui/connectors/meridio/
connectors/sharepoint/crawler-ui/connectors/...
Author: kwright
Date: Tue Mar 2 15:32:35 2010
New Revision: 918079
URL: http://svn.apache.org/viewvc?rev=918079&view=rev
Log:
Centralize all synchronized directory access to be within the ILockManager abstraction layer. This is so we can easily move from a synchronized directory architecture to a synchronization service based one.
This commit is currently untested, and has the potential to cause problems with service startup, service shutdown, and cache expiration.
Modified:
incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/editconfig.jsp
incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/headerconfig.jsp
incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/postconfig.jsp
incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/editconfig.jsp
incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/headerconfig.jsp
incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/postconfig.jsp
incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/editconfig.jsp
incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/headerconfig.jsp
incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/postconfig.jsp
incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/editconfig.jsp
incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/headerconfig.jsp
incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/postconfig.jsp
incubator/lcf/trunk/modules/connectors/webcrawler/connector/org/apache/lcf/crawler/connectors/webcrawler/TrustsDescription.java
incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentRun.java
incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentStop.java
incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/incrementalingest/IncrementalIngester.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/cachemanager/CacheManager.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfaceMySQL.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfacePostgreSQL.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/Database.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/TransactionHandle.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/IDFactory.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/ILockManager.java
incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/lockmanager/LockManager.java
incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/HopCount.java
incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobManager.java
incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobQueue.java
incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/Jobs.java
incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryConnectionManager.java
incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryHistoryManager.java
Modified: incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/editconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/editconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/editconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/editconfig.jsp Tue Mar 2 15:32:35 2010
@@ -183,7 +183,7 @@
}
%>
</table>
- <input type="button" onclick='<%="Javascript:LLAddCertificate(\""+IDFactory.make()+"\")"%>' alt="Add cert" value="Add"/>
+ <input type="button" onclick='<%="Javascript:LLAddCertificate()"%>' alt="Add cert" value="Add"/>
Certificate: <input name="llcertificate" size="50" type="file"/>
</td>
</tr>
Modified: incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/headerconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/headerconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/headerconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/headerconfig.jsp Tue Mar 2 15:32:35 2010
@@ -60,7 +60,7 @@
postForm();
}
- function LLAddCertificate(aliasName)
+ function LLAddCertificate()
{
if (editconnection.llcertificate.value == "")
{
@@ -69,7 +69,6 @@
}
else
{
- editconnection.llkeystorealias.value = aliasName;
editconnection.configop.value = "Add";
postForm();
}
Modified: incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/postconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/postconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/postconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/livelink/crawler-ui/connectors/livelink/postconfig.jsp Tue Mar 2 15:32:35 2010
@@ -99,7 +99,7 @@
}
else if (configOp.equals("Add"))
{
- String alias = variableContext.getParameter("llkeystorealias");
+ String alias = IDFactory.make(threadContext);
byte[] certificateValue = variableContext.getBinaryBytes("llcertificate");
keystoreValue = parameters.getParameter(org.apache.lcf.crawler.connectors.livelink.LiveLinkParameters.livelinkKeystore);
IKeystoreManager mgr;
Modified: incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/editconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/editconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/editconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/editconfig.jsp Tue Mar 2 15:32:35 2010
@@ -303,7 +303,7 @@
}
%>
</table>
- <input type="button" onclick='<%="Javascript:AddCertificate(\""+IDFactory.make()+"\")"%>' alt="Add cert" value="Add"/>
+ <input type="button" onclick='<%="Javascript:AddCertificate()"%>' alt="Add cert" value="Add"/>
Certificate: <input name="certificate" size="50" type="file"/>
</td>
</tr>
Modified: incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/headerconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/headerconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/headerconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/headerconfig.jsp Tue Mar 2 15:32:35 2010
@@ -163,7 +163,7 @@
postForm();
}
- function AddCertificate(aliasName)
+ function AddCertificate()
{
if (editconnection.certificate.value == "")
{
@@ -172,7 +172,6 @@
}
else
{
- editconnection.keystorealias.value = aliasName;
editconnection.configop.value = "Add";
postForm();
}
Modified: incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/postconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/postconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/postconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/authorities/meridio/postconfig.jsp Tue Mar 2 15:32:35 2010
@@ -138,7 +138,7 @@
}
else if (configOp.equals("Add"))
{
- String alias = variableContext.getParameter("keystorealias");
+ String alias = IDFactory.make(threadContext);
byte[] certificateValue = variableContext.getBinaryBytes("certificate");
keystoreValue = parameters.getParameter("MeridioKeystore");
IKeystoreManager mgr;
Modified: incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/editconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/editconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/editconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/editconfig.jsp Tue Mar 2 15:32:35 2010
@@ -253,7 +253,7 @@
}
%>
</table>
- <input type="button" onclick='<%="Javascript:AddCertificate(\""+IDFactory.make()+"\")"%>' alt="Add cert" value="Add"/>
+ <input type="button" onclick='<%="Javascript:AddCertificate()"%>' alt="Add cert" value="Add"/>
Certificate: <input name="certificate" size="50" type="file"/>
</td>
</tr>
Modified: incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/headerconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/headerconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/headerconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/headerconfig.jsp Tue Mar 2 15:32:35 2010
@@ -156,7 +156,7 @@
postForm();
}
- function AddCertificate(aliasName)
+ function AddCertificate()
{
if (editconnection.certificate.value == "")
{
@@ -165,7 +165,6 @@
}
else
{
- editconnection.keystorealias.value = aliasName;
editconnection.configop.value = "Add";
postForm();
}
Modified: incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/postconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/postconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/postconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/meridio/crawler-ui/connectors/meridio/postconfig.jsp Tue Mar 2 15:32:35 2010
@@ -133,7 +133,7 @@
}
else if (configOp.equals("Add"))
{
- String alias = variableContext.getParameter("keystorealias");
+ String alias = IDFactory.make(threadContext);
byte[] certificateValue = variableContext.getBinaryBytes("certificate");
keystoreValue = parameters.getParameter("MeridioKeystore");
IKeystoreManager mgr;
Modified: incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/editconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/editconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/editconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/editconfig.jsp Tue Mar 2 15:32:35 2010
@@ -150,7 +150,7 @@
}
%>
</table>
- <input type="button" onclick='<%="Javascript:ShpAddCertificate(\""+IDFactory.make()+"\")"%>' alt="Add cert" value="Add"/>
+ <input type="button" onclick='<%="Javascript:ShpAddCertificate()"%>' alt="Add cert" value="Add"/>
Certificate: <input name="shpcertificate" size="50" type="file"/>
</td>
</tr>
Modified: incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/headerconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/headerconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/headerconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/headerconfig.jsp Tue Mar 2 15:32:35 2010
@@ -56,7 +56,7 @@
postForm();
}
- function ShpAddCertificate(aliasName)
+ function ShpAddCertificate()
{
if (editconnection.shpcertificate.value == "")
{
@@ -65,7 +65,6 @@
}
else
{
- editconnection.shpkeystorealias.value = aliasName;
editconnection.configop.value = "Add";
postForm();
}
Modified: incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/postconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/postconfig.jsp?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/postconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/sharepoint/crawler-ui/connectors/sharepoint/postconfig.jsp Tue Mar 2 15:32:35 2010
@@ -86,7 +86,7 @@
}
else if (configOp.equals("Add"))
{
- String alias = variableContext.getParameter("shpkeystorealias");
+ String alias = IDFactory.make(threadContext);
byte[] certificateValue = variableContext.getBinaryBytes("shpcertificate");
keystoreValue = parameters.getParameter("keystore");
IKeystoreManager mgr;
Modified: incubator/lcf/trunk/modules/connectors/webcrawler/connector/org/apache/lcf/crawler/connectors/webcrawler/TrustsDescription.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/webcrawler/connector/org/apache/lcf/crawler/connectors/webcrawler/TrustsDescription.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/webcrawler/connector/org/apache/lcf/crawler/connectors/webcrawler/TrustsDescription.java (original)
+++ incubator/lcf/trunk/modules/connectors/webcrawler/connector/org/apache/lcf/crawler/connectors/webcrawler/TrustsDescription.java Tue Mar 2 15:32:35 2010
@@ -97,6 +97,7 @@
{
IKeystoreManager rval = KeystoreManagerFactory.make("");
+ int certNumber = 0;
Iterator iter = patternHash.keySet().iterator();
while (iter.hasNext())
{
@@ -113,7 +114,8 @@
int j = 0;
while (j < aliases.length)
{
- rval.addCertificate(IDFactory.make(),trustStore.getCertificate(aliases[j++]));
+ rval.addCertificate(Integer.toString(certNumber),trustStore.getCertificate(aliases[j++]));
+ certNumber++;
}
}
}
Modified: incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentRun.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentRun.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentRun.java (original)
+++ incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentRun.java Tue Mar 2 15:32:35 2010
@@ -28,6 +28,8 @@
{
public static final String _rcsid = "@(#)$Id$";
+ public static final String agentShutdownSignal = "_AGENTRUN_";
+
private AgentRun()
{
}
@@ -42,28 +44,22 @@
}
LCF.initializeEnvironment();
+ IThreadContext tc = ThreadContextFactory.make();
- // Create a file to indicate that we're running
- String synchDirectory = LCF.getProperty(LCF.synchDirectoryProperty);
- File synchFile = null;
- if (synchDirectory != null)
- {
- synchFile = new File(synchDirectory,"agentrun.file");
- // delete it if present
- synchFile.delete();
- }
try
{
- IThreadContext tc = ThreadContextFactory.make();
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ // Clear the agents shutdown signal.
+ lockManager.clearGlobalFlag(agentShutdownSignal);
System.err.println("Running...");
try
{
while (true)
{
- // See if file still there
- if (synchFile != null && synchFile.exists())
+ // Any shutdown signal yet?
+ if (lockManager.checkGlobalFlag(agentShutdownSignal))
break;
-
+
// Start whatever agents need to be started
LCF.startAgents(tc);
Modified: incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentStop.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentStop.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentStop.java (original)
+++ incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/AgentStop.java Tue Mar 2 15:32:35 2010
@@ -42,22 +42,16 @@
}
LCF.initializeEnvironment();
-
- // Create a file to indicate that we're stopping
- String synchDirectory = LCF.getProperty(LCF.synchDirectoryProperty);
- File synchFile = null;
- if (synchDirectory != null)
+ IThreadContext tc = ThreadContextFactory.make();
+ try
+ {
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ lockManager.setGlobalFlag(AgentRun.agentShutdownSignal);
+ }
+ catch (LCFException e)
{
- synchFile = new File(synchDirectory,"agentrun.file");
- try
- {
- synchFile.createNewFile();
- }
- catch (IOException e)
- {
- e.printStackTrace();
- System.exit(1);
- }
+ e.printStackTrace();
+ System.exit(1);
}
}
Modified: incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/incrementalingest/IncrementalIngester.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ incubator/lcf/trunk/modules/framework/agents/org/apache/lcf/agents/incrementalingest/IncrementalIngester.java Tue Mar 2 15:32:35 2010
@@ -1188,7 +1188,7 @@
else
map.put(authorityNameField,"");
- Long id = new Long(IDFactory.make());
+ Long id = new Long(IDFactory.make(threadContext));
map.put(idField,id);
map.put(outputConnNameField,outputConnectionName);
map.put(docKeyField,docKey);
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/cachemanager/CacheManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/cachemanager/CacheManager.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/cachemanager/CacheManager.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/cachemanager/CacheManager.java Tue Mar 2 15:32:35 2010
@@ -38,9 +38,6 @@
protected ILockManager lockManager;
protected static GeneralCache cache = new GeneralCache();
- // This is the synch directory; it's null if no cross-JVM synchronization
- protected String synchDirectory = null;
-
// This is the hash mapping transaction id's to CacheTransactionHandle objects.
// It is thread specific because transactions are thread local.
protected HashMap transactionHash = new HashMap();
@@ -49,7 +46,6 @@
throws LCFException
{
lockManager = LockManagerFactory.make(context);
- synchDirectory = LCF.getProperty(LCF.synchDirectoryProperty);
}
/** Locate or create a set of objects in the cached object pool, and/or destroy and invalidate
@@ -429,21 +425,18 @@
// Before we conclude that the object is found, if we are on a multi-JVM environment we MUST check
// the object's timestamp!!! We check it against the invalidation key file timestamps for the object.
- if (synchDirectory != null)
- {
- long createTime = cache.getObjectCreationTime(objectDescription);
- StringSet keys = cache.getObjectInvalidationKeys(objectDescription);
+ long createTime = cache.getObjectCreationTime(objectDescription);
+ StringSet keys = cache.getObjectInvalidationKeys(objectDescription);
- Iterator iter = keys.getKeys();
- while (iter.hasNext())
+ Iterator iter = keys.getKeys();
+ while (iter.hasNext())
+ {
+ String key = (String)iter.next();
+ if (hasExpired(key,createTime))
{
- String key = (String)iter.next();
- if (hasExpired(key,createTime))
- {
- // Blow away the entry in cache, since it has expired
- cache.deleteObject(objectDescription);
- return null;
- }
+ // Blow away the entry in cache, since it has expired
+ cache.deleteObject(objectDescription);
+ return null;
}
}
@@ -463,39 +456,12 @@
protected boolean hasExpired(String key, long createTime)
throws LCFException
{
-
- File fileDescription = new File(makeFilePath(key),makeFileName(key));
- long createdDate = readFile(fileDescription);
+ long createdDate = readSharedData(key);
if (createdDate == 0L)
return false;
return createdDate >= createTime;
}
- /** Create a file path given a key name.
- *@param key is the key name.
- *@return the file path.
- */
- protected String makeFilePath(String key)
- {
- int hashcode = key.hashCode();
- int outerDirNumber = (hashcode & (1023));
- int innerDirNumber = ((hashcode >> 10) & (1023));
- String fullDir = synchDirectory;
- if (fullDir.length() == 0 || !fullDir.endsWith("/"))
- fullDir = fullDir + "/";
- fullDir = fullDir + Integer.toString(outerDirNumber)+"/"+Integer.toString(innerDirNumber);
- return fullDir;
- }
-
- /** Create a file name given a key name.
- *@param key is the key name.
- *@return the file name.
- */
- protected static String makeFileName(String key)
- {
- return "cache-"+LCF.safeFileName(key);
- }
-
/** Set object's expiration and LRU.
*@param objectDescription is the description object.
*@param currentTime is the current time in milliseconds since epoch.
@@ -637,33 +603,23 @@
}
/** Perform an invalidation. Assume all appropriate locks are in place.
- * Eventually, this will be cross-cluster.
*@param keys is the set of keys to invalidate.
*/
protected void performInvalidation(StringSet keys)
throws LCFException
{
- // Finally, perform the invalidation. When support is added for
- // multi cluster, this will need to be extended to blow the cache
- // cross cluster.
+ // Finally, perform the invalidation.
if (keys != null)
{
- if (synchDirectory != null)
+ long invalidationTime = System.currentTimeMillis();
+ // Loop through all keys
+ Iterator iter = keys.getKeys();
+ while (iter.hasNext())
{
- long invalidationTime = System.currentTimeMillis();
- // Loop through all keys
- Iterator iter = keys.getKeys();
- while (iter.hasNext())
- {
- String keyName = (String)iter.next();
- String path = makeFilePath(keyName);
- // Make sure the directory exists
- (new File(path)).mkdirs();
- File fileDescription = new File(path,makeFileName(keyName));
- writeFile(fileDescription,invalidationTime);
- }
+ String keyName = (String)iter.next();
+ writeSharedData(keyName,invalidationTime);
}
cache.invalidateKeys(keys);
@@ -855,111 +811,46 @@
// Protected methods and classes
/** Read an invalidation file contents.
- *@param fileDescriptor is the descriptor.
+ *@param key is the cache key name.
*@return the invalidation time, or 0 if none.
*/
- protected static long readFile(File fileDescriptor)
+ protected long readSharedData(String key)
throws LCFException
{
+ // Read cache resource
+ byte[] cacheResourceData = lockManager.readData("cache-"+key);
+ if (cacheResourceData == null)
+ return 0L;
try
{
- FileReader fr = new FileReader(fileDescriptor);
- try
- {
- BufferedReader x = new BufferedReader(fr);
- try
- {
- StringBuffer sb = new StringBuffer();
- while (true)
- {
- int rval = x.read();
- if (rval == -1)
- break;
- sb.append((char)rval);
- }
- return new Long(sb.toString()).longValue();
- }
- catch (InterruptedIOException e)
- {
- throw new LCFException("Interrupted",e,LCFException.INTERRUPTED);
- }
- catch (IOException e)
- {
- Logging.cache.error("Could not read from lock file: '"+fileDescriptor.toString()+"'",e);
- // Don't fail hard or there is no way to recover
- throw e;
- }
- finally
- {
- x.close();
- }
- }
- catch (InterruptedIOException e)
- {
- throw new LCFException("Interrupted",e,LCFException.INTERRUPTED);
- }
- catch (IOException e)
- {
- Logging.cache.error("Could not read from lock file: '"+fileDescriptor.toString()+"'",e);
- // Don't fail hard or there is no way to recover
- throw e;
- }
- finally
- {
- fr.close();
- }
- }
- catch (InterruptedIOException e)
- {
- throw new LCFException("Interrupted",e,LCFException.INTERRUPTED);
+ String expiration = new String(cacheResourceData,"utf-8");
+ return new Long(expiration).longValue();
}
- catch (IOException e)
+ catch (UnsupportedEncodingException e)
{
- return 0L;
+ throw new LCFException(e.getMessage(),e);
}
-
}
/** Write the invalidation file contents.
- *@param fileDescriptor is the descriptor.
+ *@param key is the cache key name.
*@param value is the invalidation timestamp.
*/
- protected static void writeFile(File fileDescriptor, long value)
+ protected void writeSharedData(String key, long value)
throws LCFException
{
- try
+ if (value == 0L)
+ lockManager.writeData(key,null);
+ else
{
- if (value == 0L)
- fileDescriptor.delete();
- else
+ try
{
- FileWriter fw = new FileWriter(fileDescriptor);
- try
- {
- BufferedWriter x = new BufferedWriter(fw);
- try
- {
- x.write(Long.toString(value));
- }
- finally
- {
- x.close();
- }
- }
- finally
- {
- fw.close();
- }
+ lockManager.writeData(key,Long.toString(value).getBytes("utf-8"));
+ }
+ catch (UnsupportedEncodingException e)
+ {
+ throw new LCFException(e.getMessage(),e);
}
- }
- catch (InterruptedIOException e)
- {
- throw new LCFException("Interrupted",e,LCFException.INTERRUPTED);
- }
- catch (IOException e)
- {
- // Hard failure is called for
- throw new LCFException("Cache system failure",e);
}
}
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfaceMySQL.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfaceMySQL.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfaceMySQL.java Tue Mar 2 15:32:35 2010
@@ -25,12 +25,14 @@
{
public static final String _rcsid = "@(#)$Id$";
+ protected IThreadContext context;
protected IDatabase database;
protected String cacheKey;
public DBInterfaceMySQL(IThreadContext tc, String databaseName, String userName, String password)
throws LCFException
{
+ this.context = tc;
if (databaseName == null)
databaseName = "mysql";
database = DatabaseFactory.make(tc,databaseName,userName,password);
@@ -314,7 +316,7 @@
if (indexName == null)
// Build an index name
- indexName = "I"+IDFactory.make();
+ indexName = "I"+IDFactory.make(context);
StringBuffer queryBuffer = new StringBuffer("CREATE ");
if (description.getIsUnique())
queryBuffer.append("UNIQUE ");
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfacePostgreSQL.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/DBInterfacePostgreSQL.java Tue Mar 2 15:32:35 2010
@@ -26,6 +26,7 @@
{
public static final String _rcsid = "@(#)$Id$";
+ protected IThreadContext context;
protected IDatabase database;
protected String cacheKey;
// Postgresql serializable transactions are broken in that transactions that occur within them do not in fact work properly.
@@ -41,6 +42,7 @@
public DBInterfacePostgreSQL(IThreadContext tc, String databaseName, String userName, String password)
throws LCFException
{
+ this.context = tc;
if (databaseName == null)
databaseName = "template1";
database = DatabaseFactory.make(tc,databaseName,userName,password);
@@ -409,7 +411,7 @@
if (indexName == null)
// Build an index name
- indexName = "I"+IDFactory.make();
+ indexName = "I"+IDFactory.make(context);
StringBuffer queryBuffer = new StringBuffer("CREATE ");
if (description.getIsUnique())
queryBuffer.append("UNIQUE ");
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/Database.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/Database.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/Database.java Tue Mar 2 15:32:35 2010
@@ -169,7 +169,7 @@
// at the database implementation layer, which will incidentally cause any delayed transactions to actually be starte.d
String enclosingID = (th==null)?null:th.getTransactionID();
delayedTransactionDepth++;
- th = new TransactionHandle(th,transactionType);
+ th = new TransactionHandle(context,th,transactionType);
cacheManager.startTransaction(th.getTransactionID(),enclosingID);
doRollback = false;
}
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/TransactionHandle.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/TransactionHandle.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/TransactionHandle.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/database/TransactionHandle.java Tue Mar 2 15:32:35 2010
@@ -35,11 +35,11 @@
protected String transactionID;
protected int transactionType;
- public TransactionHandle(TransactionHandle previousTransaction, int transactionType)
+ public TransactionHandle(IThreadContext tc, TransactionHandle previousTransaction, int transactionType)
throws LCFException
{
// Grab a unique ID
- transactionID = IDFactory.make();
+ transactionID = IDFactory.make(tc);
this.previousTransaction = previousTransaction;
this.transactionType = transactionType;
}
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/IDFactory.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/IDFactory.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/IDFactory.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/IDFactory.java Tue Mar 2 15:32:35 2010
@@ -23,183 +23,84 @@
import java.io.*;
import java.util.*;
+/** The purpose of this class is to create global unique identifiers. For performance, every JVM has a local pool of identifiers as well
+* as there being a global system of identifier creation, which is also resilient against entire system restarts.
+*/
public class IDFactory
{
public static final String _rcsid = "@(#)$Id$";
- private static long _id = 0L;
- private static final Integer _proplock = new Integer(0);
- private static boolean propertyChecked = false;
- private static File idFile = null;
- private static File idLock = null;
private static ArrayList idPool = new ArrayList();
// The id algorithm depends on the clock. We don't want to fetch too many; we'll
// run the risk of a restart in the non-synchronized case beating the clock.
private final static int poolSize = 100;
+ /** This is the critical section name */
+ private final static String criticalSectionName = "_IDFACTORY_";
+ /** This is the global lock name */
+ private final static String globalLockName = "_IDFACTORY_";
+ /** This is the name of the global ID data object */
+ private final static String globalIDDataName = "_IDFACTORY_";
+
private IDFactory()
{
}
- public static String make()
+ public static String make(IThreadContext tc)
throws LCFException
{
- synchronized (_proplock)
+ ILockManager lockManager = LockManagerFactory.make(tc);
+ // Enter critical section before we look at the pool
+ lockManager.enterWriteCriticalSection(criticalSectionName);
+ try
{
- if (propertyChecked == false)
- {
- String synchDirectory = LCF.getProperty(LCF.synchDirectoryProperty);
- if (synchDirectory != null)
- {
- idFile = new File(synchDirectory,"idfile.file");
- idLock = new File(synchDirectory,"idfile.lock");
- }
- propertyChecked = true;
- }
- }
-
- // See if there's anything in the pool
- synchronized (idPool)
- {
- if (idPool.size() > 0)
- return (String)idPool.remove(idPool.size()-1);
-
- // Need to fill the pool.
- // If synchronized, we must lock first
- if (idLock != null)
+ // check the pool
+ if (idPool.size() == 0)
{
+ // Pool was empty. We need to grab more id's from the global resource.
+ lockManager.enterWriteLock(globalLockName);
try
{
- while (true)
+ // Read shared data
+ byte[] idData = lockManager.readData(globalIDDataName);
+ long _id;
+ if (idData == null)
+ _id = 0L;
+ else
+ _id = new Long(new String(idData,"utf-8")).longValue();
+
+ int i = 0;
+ while (i < poolSize)
{
- try
+ long newid = System.currentTimeMillis();
+ if (newid <= _id)
{
- if (idLock.createNewFile())
- break;
+ newid = _id + 1;
}
- catch (InterruptedIOException e)
- {
- throw new LCFException("Interrupted: "+e.getMessage(),e,LCFException.INTERRUPTED);
- }
- catch (IOException e)
- {
- // same as returning false above.
- }
- LCF.sleep(10);
+ _id = newid;
+ idPool.add(Long.toString(newid));
+ i++;
}
- }
- catch (InterruptedException e)
- {
- throw new LCFException("Interrupted: "+e.getMessage(),e,LCFException.INTERRUPTED);
- }
- }
- try
- {
- // Get the last id used systemwide
- if (idFile != null)
- {
- try
- {
- FileReader fr = new FileReader(idFile);
- try
- {
- BufferedReader x = new BufferedReader(fr);
- try
- {
- StringBuffer sb = new StringBuffer();
- while (true)
- {
- int rval = x.read();
- if (rval == -1)
- break;
- sb.append((char)rval);
- }
- _id = new Long(sb.toString()).longValue();
- }
- catch (NumberFormatException e)
- {
- // This should never happen, but it does for reasons as yet unknown on Linux.
- Logging.misc.error("Lost ability to read id file; resetting");
- _id = 0L;
- }
- catch (IOException e)
- {
- throw new LCFException("Could not read from id file: '"+idFile.toString()+"'",e);
- }
- finally
- {
- x.close();
- }
- }
- catch (IOException e)
- {
- throw new LCFException("Could not read from id file: '"+idFile.toString()+"'",e);
- }
- finally
- {
- fr.close();
- }
- }
- catch (IOException e)
- {
- _id = 0L;
- }
+ lockManager.writeData(globalIDDataName,Long.toString(_id).getBytes("utf-8"));
}
-
- int i = 0;
- while (i < poolSize)
+ catch (UnsupportedEncodingException e)
{
- long newid = System.currentTimeMillis();
- if (newid <= _id)
- {
- newid = _id + 1;
- }
- _id = newid;
- idPool.add(Long.toString(newid));
- i++;
+ throw new LCFException(e.getMessage(),e);
}
-
- // Write the updated _id value into the file
- if (idFile != null)
+ finally
{
- try
- {
- FileWriter fw = new FileWriter(idFile);
- try
- {
- BufferedWriter x = new BufferedWriter(fw);
- try
- {
- x.write(Long.toString(_id));
- }
- finally
- {
- x.close();
- }
- }
- finally
- {
- fw.close();
- }
- }
- catch (IOException e)
- {
- // Hard failure
- throw new LCFException("Can't write id file",e);
- }
-
+ lockManager.leaveWriteLock(globalLockName);
}
}
- finally
- {
- if (idLock != null)
- idLock.delete();
- }
-
return (String)idPool.remove(idPool.size()-1);
}
+ finally
+ {
+ lockManager.leaveWriteCriticalSection(criticalSectionName);
+ }
+
}
}
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/ILockManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/ILockManager.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/ILockManager.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/ILockManager.java Tue Mar 2 15:32:35 2010
@@ -19,12 +19,49 @@
package org.apache.lcf.core.interfaces;
-/** The lock manager manages locks across all threads and JVMs and cluster members
+/** The lock manager manages locks across all threads and JVMs and cluster members. It also
+* manages shared data, which is not necessarily atomic and should be protected by locks.
*/
public interface ILockManager
{
public static final String _rcsid = "@(#)$Id$";
+ /** Raise a flag. Use this method to assert a condition, or send a global signal. The flag will be reset when the
+ * entire system is restarted.
+ *@param flagName is the name of the flag to set.
+ */
+ public void setGlobalFlag(String flagName)
+ throws LCFException;
+
+ /** Clear a flag. Use this method to clear a condition, or retract a global signal.
+ *@param flagName is the name of the flag to clear.
+ */
+ public void clearGlobalFlag(String flagName)
+ throws LCFException;
+
+ /** Check the condition of a specified flag.
+ *@param flagName is the name of the flag to check.
+ *@return true if the flag is set, false otherwise.
+ */
+ public boolean checkGlobalFlag(String flagName)
+ throws LCFException;
+
+ /** Read data from a shared data resource. Use this method to read any existing data, or get a null back if there is no such resource.
+ * Note well that this is not necessarily an atomic operation, and it must thus be protected by a lock.
+ *@param resourceName is the global name of the resource.
+ *@return a byte array containing the data, or null.
+ */
+ public byte[] readData(String resourceName)
+ throws LCFException;
+
+ /** Write data to a shared data resource. Use this method to write a body of data into a shared resource.
+ * Note well that this is not necessarily an atomic operation, and it must thus be protected by a lock.
+ *@param resourceName is the global name of the resource.
+ *@param data is the byte array containing the data. Pass null if you want to delete the resource completely.
+ */
+ public void writeData(String resourceName, byte[] data)
+ throws LCFException;
+
/** Wait for a time before retrying a lock. Use this method to wait
* after a LockException has been thrown. )If this is not done, the application
* will wind up busy waiting.)
Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/lockmanager/LockManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/lockmanager/LockManager.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/lockmanager/LockManager.java (original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/lockmanager/LockManager.java Tue Mar 2 15:32:35 2010
@@ -22,6 +22,7 @@
import org.apache.lcf.core.system.Logging;
import org.apache.lcf.core.system.LCF;
import java.util.*;
+import java.io.*;
/** The lock manager manages locks across all threads and JVMs and cluster members. There should be no more than ONE
* instance of this class per thread!!! The factory should enforce this.
@@ -52,36 +53,135 @@
synchDirectory = LCF.getProperty(LCF.synchDirectoryProperty);
}
- protected LocalLock getLocalLock(String lockKey)
+
+ /** Raise a flag. Use this method to assert a condition, or send a global signal. The flag will be reset when the
+ * entire system is restarted.
+ *@param flagName is the name of the flag to set.
+ */
+ public void setGlobalFlag(String flagName)
+ throws LCFException
{
- LocalLock ll = (LocalLock)localLocks.get(lockKey);
- if (ll == null)
+ String resourceName = "flag-" + flagName;
+ String path = makeFilePath(resourceName);
+ (new File(path)).mkdirs();
+ File f = new File(path,LCF.safeFileName(resourceName));
+ try
{
- ll = new LocalLock();
- localLocks.put(lockKey,ll);
+ f.createNewFile();
+ }
+ catch (InterruptedIOException e)
+ {
+ throw new LCFException("Interrupted: "+e.getMessage(),e,LCFException.INTERRUPTED);
+ }
+ catch (IOException e)
+ {
+ throw new LCFException(e.getMessage(),e);
}
- return ll;
}
- protected void releaseLocalLock(String lockKey)
+ /** Clear a flag. Use this method to clear a condition, or retract a global signal.
+ *@param flagName is the name of the flag to clear.
+ */
+ public void clearGlobalFlag(String flagName)
+ throws LCFException
{
- localLocks.remove(lockKey);
+ String resourceName = "flag-" + flagName;
+ File f = new File(makeFilePath(resourceName),LCF.safeFileName(resourceName));
+ f.delete();
+ }
+
+ /** Check the condition of a specified flag.
+ *@param flagName is the name of the flag to check.
+ *@return true if the flag is set, false otherwise.
+ */
+ public boolean checkGlobalFlag(String flagName)
+ throws LCFException
+ {
+ String resourceName = "flag-" + flagName;
+ File f = new File(makeFilePath(resourceName),LCF.safeFileName(resourceName));
+ return f.exists();
}
- protected LocalLock getLocalSection(String sectionKey)
+ /** Read data from a shared data resource. Use this method to read any existing data, or get a null back if there is no such resource.
+ * Note well that this is not necessarily an atomic operation, and it must thus be protected by a lock.
+ *@param resourceName is the global name of the resource.
+ *@return a byte array containing the data, or null.
+ */
+ public byte[] readData(String resourceName)
+ throws LCFException
{
- LocalLock ll = (LocalLock)localSections.get(sectionKey);
- if (ll == null)
+ File f = new File(makeFilePath(resourceName),LCF.safeFileName(resourceName));
+ try
{
- ll = new LocalLock();
- localSections.put(sectionKey,ll);
+ InputStream is = new FileInputStream(f);
+ try
+ {
+ ByteArrayBuffer bab = new ByteArrayBuffer();
+ while (true)
+ {
+ int x = is.read();
+ if (x == -1)
+ break;
+ bab.add((byte)x);
+ }
+ return bab.toArray();
+ }
+ finally
+ {
+ is.close();
+ }
+ }
+ catch (FileNotFoundException e)
+ {
+ return null;
+ }
+ catch (InterruptedIOException e)
+ {
+ throw new LCFException("Interrupted: "+e.getMessage(),e,LCFException.INTERRUPTED);
+ }
+ catch (IOException e)
+ {
+ throw new LCFException("IO exception: "+e.getMessage(),e);
}
- return ll;
}
-
- protected void releaseLocalSection(String sectionKey)
+
+ /** Write data to a shared data resource. Use this method to write a body of data into a shared resource.
+ * Note well that this is not necessarily an atomic operation, and it must thus be protected by a lock.
+ *@param resourceName is the global name of the resource.
+ *@param data is the byte array containing the data. Pass null if you want to delete the resource completely.
+ */
+ public void writeData(String resourceName, byte[] data)
+ throws LCFException
{
- localSections.remove(sectionKey);
+ try
+ {
+ String path = makeFilePath(resourceName);
+ // Make sure the directory exists
+ (new File(path)).mkdirs();
+ File f = new File(path,LCF.safeFileName(resourceName));
+ if (data == null)
+ {
+ f.delete();
+ return;
+ }
+ FileOutputStream os = new FileOutputStream(f);
+ try
+ {
+ os.write(data,0,data.length);
+ }
+ finally
+ {
+ os.close();
+ }
+ }
+ catch (InterruptedIOException e)
+ {
+ throw new LCFException("Interrupted: "+e.getMessage(),e,LCFException.INTERRUPTED);
+ }
+ catch (IOException e)
+ {
+ throw new LCFException("IO exception: "+e.getMessage(),e);
+ }
}
/** Wait for a time before retrying a lock.
@@ -1544,6 +1644,38 @@
}
}
+ protected LocalLock getLocalLock(String lockKey)
+ {
+ LocalLock ll = (LocalLock)localLocks.get(lockKey);
+ if (ll == null)
+ {
+ ll = new LocalLock();
+ localLocks.put(lockKey,ll);
+ }
+ return ll;
+ }
+
+ protected void releaseLocalLock(String lockKey)
+ {
+ localLocks.remove(lockKey);
+ }
+
+ protected LocalLock getLocalSection(String sectionKey)
+ {
+ LocalLock ll = (LocalLock)localSections.get(sectionKey);
+ if (ll == null)
+ {
+ ll = new LocalLock();
+ localSections.put(sectionKey,ll);
+ }
+ return ll;
+ }
+
+ protected void releaseLocalSection(String sectionKey)
+ {
+ localSections.remove(sectionKey);
+ }
+
/** Process inbound locks into a sorted vector of most-restrictive unique locks
*/
protected LockDescription[] getSortedUniqueLocks(String[] readLocks, String[] writeNonExLocks,
@@ -1621,6 +1753,21 @@
return rval;
}
+ /** Create a file path given a key name.
+ *@param key is the key name.
+ *@return the file path.
+ */
+ protected String makeFilePath(String key)
+ {
+ int hashcode = key.hashCode();
+ int outerDirNumber = (hashcode & (1023));
+ int innerDirNumber = ((hashcode >> 10) & (1023));
+ String fullDir = synchDirectory;
+ if (fullDir.length() == 0 || !fullDir.endsWith("/"))
+ fullDir = fullDir + "/";
+ fullDir = fullDir + Integer.toString(outerDirNumber)+"/"+Integer.toString(innerDirNumber);
+ return fullDir;
+ }
protected class LockDescription
{
@@ -1705,4 +1852,37 @@
writeCount--;
}
}
+
+ protected static final int BASE_SIZE = 128;
+
+ protected static class ByteArrayBuffer
+ {
+ protected byte[] buffer;
+ protected int length;
+
+ public ByteArrayBuffer()
+ {
+ buffer = new byte[BASE_SIZE];
+ length = 0;
+ }
+
+ public void add(byte b)
+ {
+ if (length == buffer.length)
+ {
+ byte[] oldbuffer = buffer;
+ buffer = new byte[length * 2];
+ System.arraycopy(oldbuffer,0,buffer,0,length);
+ }
+ buffer[length++] = b;
+ }
+
+ public byte[] toArray()
+ {
+ byte[] rval = new byte[length];
+ System.arraycopy(buffer,0,rval,0,length);
+ return rval;
+ }
+ }
+
}
Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/HopCount.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/HopCount.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/HopCount.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/HopCount.java Tue Mar 2 15:32:35 2010
@@ -138,13 +138,17 @@
/** Hop "delete" dependencies manager */
protected HopDeleteDeps deleteDepsManager;
+ /** Thread context */
+ protected IThreadContext threadContext;
+
/** Constructor.
*@param database is the database handle.
*/
- public HopCount(IDBInterface database)
+ public HopCount(IThreadContext tc, IDBInterface database)
throws LCFException
{
super(database,"hopcount");
+ this.threadContext = tc;
intrinsicLinkManager = new IntrinsicLink(database);
deleteDepsManager = new HopDeleteDeps(database);
}
@@ -877,7 +881,7 @@
DeleteDependency dd = new DeleteDependency(linkType,documentIDHash,sourceDocumentIDHash);
// Build a new answer, based on the starting answer and the kind of link this is.
map.clear();
- Long hopCountID = new Long(IDFactory.make());
+ Long hopCountID = new Long(IDFactory.make(threadContext));
map.put(idField,hopCountID);
map.put(parentIDHashField,q.getDocumentIdentifierHash());
map.put(linkTypeField,q.getLinkType());
@@ -1745,7 +1749,7 @@
{
// We do NOT expect there to already be a cached entry! If there is, we've screwed
// up somehow, and it's a bug.
- Long id = new Long(IDFactory.make());
+ Long id = new Long(IDFactory.make(threadContext));
map.put(idField,id);
map.put(jobIDField,jobID);
Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobManager.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobManager.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobManager.java Tue Mar 2 15:32:35 2010
@@ -59,8 +59,8 @@
this.database = database;
this.threadContext = threadContext;
jobs = new Jobs(threadContext,database);
- jobQueue = new JobQueue(database);
- hopCount = new HopCount(database);
+ jobQueue = new JobQueue(threadContext,database);
+ hopCount = new HopCount(threadContext,database);
carryDown = new Carrydown(database);
eventManager = new EventManager(database);
outputMgr = OutputConnectionManagerFactory.make(threadContext);
Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobQueue.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobQueue.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/JobQueue.java Tue Mar 2 15:32:35 2010
@@ -132,13 +132,17 @@
/** Prerequisite event manager */
protected PrereqEventManager prereqEventManager;
+ /** Thread context */
+ protected IThreadContext threadContext;
+
/** Constructor.
*@param database is the database handle.
*/
- public JobQueue(IDBInterface database)
+ public JobQueue(IThreadContext tc, IDBInterface database)
throws LCFException
{
super(database,"jobqueue");
+ this.threadContext = tc;
prereqEventManager = new PrereqEventManager(database);
}
@@ -833,7 +837,7 @@
{
// No prerequisites should be possible at this point.
HashMap map = new HashMap();
- Long recordID = new Long(IDFactory.make());
+ Long recordID = new Long(IDFactory.make(threadContext));
map.put(idField,recordID);
if (desiredExecuteTime == -1L)
map.put(checkTimeField,new Long(0L));
@@ -1146,7 +1150,7 @@
throws LCFException
{
HashMap map = new HashMap();
- Long recordID = new Long(IDFactory.make());
+ Long recordID = new Long(IDFactory.make(threadContext));
map.put(idField,recordID);
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/Jobs.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/Jobs.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/jobs/Jobs.java Tue Mar 2 15:32:35 2010
@@ -184,6 +184,8 @@
protected IOutputConnectionManager outputMgr;
protected IRepositoryConnectionManager connectionMgr;
+ protected IThreadContext threadContext;
+
/** Constructor.
*@param database is the database handle.
*/
@@ -191,6 +193,7 @@
throws LCFException
{
super(database,"jobs");
+ this.threadContext = threadContext;
scheduleManager = new ScheduleManager(threadContext,database);
hopFilterManager = new HopFilterManager(threadContext,database);
cacheManager = CacheManagerFactory.make(threadContext);
@@ -497,7 +500,7 @@
{
JobDescription rval = new JobDescription();
rval.setIsNew(true);
- rval.setID(new Long(IDFactory.make()));
+ rval.setID(new Long(IDFactory.make(threadContext)));
return rval;
}
Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryConnectionManager.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryConnectionManager.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryConnectionManager.java Tue Mar 2 15:32:35 2010
@@ -64,7 +64,7 @@
{
super(database,"repoconnections");
- historyManager = new RepositoryHistoryManager(database);
+ historyManager = new RepositoryHistoryManager(threadContext,database);
throttleSpecManager = new ThrottleSpecManager(database);
cacheManager = CacheManagerFactory.make(threadContext);
this.threadContext = threadContext;
Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryHistoryManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryHistoryManager.java?rev=918079&r1=918078&r2=918079&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryHistoryManager.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/repository/RepositoryHistoryManager.java Tue Mar 2 15:32:35 2010
@@ -42,13 +42,17 @@
/** Counter for kicking off analyze */
protected static AnalyzeTracker tracker = new AnalyzeTracker();
+ /** Thread context */
+ protected IThreadContext threadContext;
+
/** Constructor.
*@param database is the database instance.
*/
- public RepositoryHistoryManager(IDBInterface database)
+ public RepositoryHistoryManager(IThreadContext tc, IDBInterface database)
throws LCFException
{
super(database,"repohistory");
+ this.threadContext = tc;
}
/** Install or upgrade the table.
@@ -149,7 +153,7 @@
String entityIdentifier, String resultCode, String resultDescription)
throws LCFException
{
- Long id = new Long(IDFactory.make());
+ Long id = new Long(IDFactory.make(threadContext));
HashMap map = new HashMap();
map.put(idField,id);
map.put(ownerNameField,connectionName);