You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@manifoldcf.apache.org by Adrian Conlon <Ad...@arup.com> on 2014/01/08 17:51:18 UTC
Lock Warnings in ManifoldCF logs
Hi All,
I'm getting large numbers of locking warnings in my logs on a Windows server (normally I run under Ubuntu/Linux). Eventually the agent service stops working.
Anyway, here's a couple of sample exceptions from the log:
WARN 2014-01-07 02:04:47,740 (http-bio-8080-exec-20) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\32\542\lock-_Cache_OUTPUTCONNECTION_Oasys&58! Output connection.lock' failed: Access is denied
java.io.IOException: Access is denied
at java.io.WinNTFileSystem.createFileExclusively(Native Method)
at java.io.File.createNewFile(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLockNoWait(LockObject.java:439)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLock(LockObject.java:401)
at org.apache.manifoldcf.core.lockmanager.LockManager.enterLocks(LockManager.java:924)
at org.apache.manifoldcf.core.cachemanager.CacheManager.enterCache(CacheManager.java:278)
at org.apache.manifoldcf.core.cachemanager.CacheManager.findObjectsAndExecute(CacheManager.java:98)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.loadMultiple(OutputConnectionManager.java:256)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.load(OutputConnectionManager.java:233)
at org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadOutputConnection(ManifoldCF.java:2122)
at org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3191)
at org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
at org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock' failed: The system cannot find the path specified
java.io.IOException: The system cannot find the path specified
at java.io.WinNTFileSystem.createFileExclusively(Native Method)
at java.io.File.createNewFile(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
at org.apache.manifoldcf.core.lockmanager.LockObject.leaveReadLock(LockObject.java:489)
at org.apache.manifoldcf.core.lockmanager.LockManager.leaveReadLock(LockManager.java:752)
at org.apache.manifoldcf.core.lockmanager.LockManager.leaveLocks(LockManager.java:1216)
at org.apache.manifoldcf.core.cachemanager.CacheManager.commitTransaction(CacheManager.java:761)
at org.apache.manifoldcf.core.database.Database.endTransaction(Database.java:395)
at org.apache.manifoldcf.core.database.DBInterfacePostgreSQL.endTransaction(DBInterfacePostgreSQL.java:1161)
at org.apache.manifoldcf.core.database.BaseTable.endTransaction(BaseTable.java:277)
at org.apache.manifoldcf.crawler.jobs.Jobs.getAll(Jobs.java:538)
at org.apache.manifoldcf.crawler.jobs.JobManager.getAllJobs(JobManager.java:486)
at org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadJobs(ManifoldCF.java:1753)
at org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3069)
at org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
at org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
Does anyone have any ideas on what might be causing this? I note that although my AWS Ubuntu instance doesn't fail (or generate warnings), the syncharea does seem to get awfully large. Is it the intention that the syncharea should be self-cleansing?
Adrian
p.s.
I should note that my (home-grown) ManifoldCF controller software generates large amounts of ManifoldCF jobs, and is always tinkering with them (starting, stopping, modifying, etc.).
____________________________________________________________
Electronic mail messages entering and leaving Arup business
systems are scanned for acceptability of content and viruses
RE: Lock Warnings in ManifoldCF logs
Posted by Adrian Conlon <Ad...@arup.com>.
Hi Karl,
Well, the good news is we've got rid of pretty much *all* of the lock file warnings by ensuring the all ManifoldCF programs are running as the same user.
The bad news is that we're left with a lock error that shuts down the agents process:
ERROR 2014-01-09 23:43:47,560 (Worker thread '17') - Couldn't write to lock file; disk may be full. Shutting down process; locks may be left dangling. You must cleanup before restarting.
java.io.FileNotFoundException: C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\32\542\lock-_Cache_OUTPUTCONNECTION_Oasys&58! Output connection.file (Access is denied)
at java.io.FileOutputStream.open(Native Method)
at java.io.FileOutputStream.<init>(Unknown Source)
at java.io.FileOutputStream.<init>(Unknown Source)
at java.io.FileWriter.<init>(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.writeFile(LockObject.java:732)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLockNoWait(LockObject.java:449)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLock(LockObject.java:401)
at org.apache.manifoldcf.core.lockmanager.LockManager.enterLocks(LockManager.java:924)
at org.apache.manifoldcf.core.cachemanager.CacheManager.enterCache(CacheManager.java:278)
at org.apache.manifoldcf.core.cachemanager.CacheManager.findObjectsAndExecute(CacheManager.java:98)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.loadMultiple(OutputConnectionManager.java:256)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.load(OutputConnectionManager.java:233)
at org.apache.manifoldcf.agents.incrementalingest.IncrementalIngester.getOutputDescription(IncrementalIngester.java:306)
at org.apache.manifoldcf.crawler.system.WorkerThread.run(WorkerThread.java:306)
This error looks like some kind of resource management limit issue, so I'll look into this (the disk isn't full). If you've got any further ideas, that'd be useful...
Adrian
From: Adrian Conlon [mailto:Adrian.Conlon@arup.com]
Sent: 08 January 2014 19:16
To: user@manifoldcf.apache.org
Subject: RE: Lock Warnings in ManifoldCF logs
Hi Karl,
Thanks for the reply: that does make some sense, I haven't bothered ensuring my agents are running under the same account as the servlet container on Windows. On Linux, I *did* sort this out just because it was the most straightforward approach.
I'll sort out my Windows agents process (might take a day or two before I get around to this) and reply back to this thread.
Thanks again for the info,
Adrian
From: Karl Wright [mailto:daddywri@gmail.com]
Sent: 08 January 2014 17:51
To: user@manifoldcf.apache.org<ma...@manifoldcf.apache.org>
Subject: Re: Lock Warnings in ManifoldCF logs
Hi Adrian,
"WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock' failed: The system cannot find the path specified
java.io.IOException: The system cannot find the path specified
at java.io.WinNTFileSystem.
createFileExclusively(Native Method)"
This does not have anything to do with your controller software. What it is complaining about is that it cannot create a file. There are a number of possible reasons for this.
(1) You may be out of system resources
(2) You may have run ManifoldCF scripts as an incorrect user, causing directories to be created which other manifoldcf processes don't have write access to
(3) Windows may have gotten itself into a funky state
My suggestion is to first of all make SURE your ManifoldCF processes are all being executed by the same user. THEN perform the lock clean sequence:
- shut down all manifoldcf processes
- run the lockclean.bat script
- start your manifoldcf processes again
Thanks,
Karl
On Wed, Jan 8, 2014 at 11:51 AM, Adrian Conlon <Ad...@arup.com>> wrote:
Hi All,
I'm getting large numbers of locking warnings in my logs on a Windows server (normally I run under Ubuntu/Linux). Eventually the agent service stops working.
Anyway, here's a couple of sample exceptions from the log:
WARN 2014-01-07 02:04:47,740 (http-bio-8080-exec-20) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\32\542\lock-_Cache_OUTPUTCONNECTION_Oasys&58! Output connection.lock' failed: Access is denied
java.io.IOException: Access is denied
at java.io.WinNTFileSystem.createFileExclusively(Native Method)
at java.io.File.createNewFile(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLockNoWait(LockObject.java:439)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLock(LockObject.java:401)
at org.apache.manifoldcf.core.lockmanager.LockManager.enterLocks(LockManager.java:924)
at org.apache.manifoldcf.core.cachemanager.CacheManager.enterCache(CacheManager.java:278)
at org.apache.manifoldcf.core.cachemanager.CacheManager.findObjectsAndExecute(CacheManager.java:98)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.loadMultiple(OutputConnectionManager.java:256)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.load(OutputConnectionManager.java:233)
at org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadOutputConnection(ManifoldCF.java:2122)
at org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3191)
at org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
at org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock' failed: The system cannot find the path specified
java.io.IOException: The system cannot find the path specified
at java.io.WinNTFileSystem.createFileExclusively(Native Method)
at java.io.File.createNewFile(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
at org.apache.manifoldcf.core.lockmanager.LockObject.leaveReadLock(LockObject.java:489)
at org.apache.manifoldcf.core.lockmanager.LockManager.leaveReadLock(LockManager.java:752)
at org.apache.manifoldcf.core.lockmanager.LockManager.leaveLocks(LockManager.java:1216)
at org.apache.manifoldcf.core.cachemanager.CacheManager.commitTransaction(CacheManager.java:761)
at org.apache.manifoldcf.core.database.Database.endTransaction(Database.java:395)
at org.apache.manifoldcf.core.database.DBInterfacePostgreSQL.endTransaction(DBInterfacePostgreSQL.java:1161)
at org.apache.manifoldcf.core.database.BaseTable.endTransaction(BaseTable.java:277)
at org.apache.manifoldcf.crawler.jobs.Jobs.getAll(Jobs.java:538)
at org.apache.manifoldcf.crawler.jobs.JobManager.getAllJobs(JobManager.java:486)
at org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadJobs(ManifoldCF.java:1753)
at org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3069)
at org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
at org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
Does anyone have any ideas on what might be causing this? I note that although my AWS Ubuntu instance doesn't fail (or generate warnings), the syncharea does seem to get awfully large. Is it the intention that the syncharea should be self-cleansing?
Adrian
p.s.
I should note that my (home-grown) ManifoldCF controller software generates large amounts of ManifoldCF jobs, and is always tinkering with them (starting, stopping, modifying, etc.).
____________________________________________________________
Electronic mail messages entering and leaving Arup business
systems are scanned for acceptability of content and viruses
RE: Lock Warnings in ManifoldCF logs
Posted by Adrian Conlon <Ad...@arup.com>.
Hi Karl,
Thanks for the reply: that does make some sense, I haven't bothered ensuring my agents are running under the same account as the servlet container on Windows. On Linux, I *did* sort this out just because it was the most straightforward approach.
I'll sort out my Windows agents process (might take a day or two before I get around to this) and reply back to this thread.
Thanks again for the info,
Adrian
From: Karl Wright [mailto:daddywri@gmail.com]
Sent: 08 January 2014 17:51
To: user@manifoldcf.apache.org
Subject: Re: Lock Warnings in ManifoldCF logs
Hi Adrian,
"WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock' failed: The system cannot find the path specified
java.io.IOException: The system cannot find the path specified
at java.io.WinNTFileSystem.
createFileExclusively(Native Method)"
This does not have anything to do with your controller software. What it is complaining about is that it cannot create a file. There are a number of possible reasons for this.
(1) You may be out of system resources
(2) You may have run ManifoldCF scripts as an incorrect user, causing directories to be created which other manifoldcf processes don't have write access to
(3) Windows may have gotten itself into a funky state
My suggestion is to first of all make SURE your ManifoldCF processes are all being executed by the same user. THEN perform the lock clean sequence:
- shut down all manifoldcf processes
- run the lockclean.bat script
- start your manifoldcf processes again
Thanks,
Karl
On Wed, Jan 8, 2014 at 11:51 AM, Adrian Conlon <Ad...@arup.com>> wrote:
Hi All,
I'm getting large numbers of locking warnings in my logs on a Windows server (normally I run under Ubuntu/Linux). Eventually the agent service stops working.
Anyway, here's a couple of sample exceptions from the log:
WARN 2014-01-07 02:04:47,740 (http-bio-8080-exec-20) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\32\542\lock-_Cache_OUTPUTCONNECTION_Oasys&58! Output connection.lock' failed: Access is denied
java.io.IOException: Access is denied
at java.io.WinNTFileSystem.createFileExclusively(Native Method)
at java.io.File.createNewFile(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLockNoWait(LockObject.java:439)
at org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLock(LockObject.java:401)
at org.apache.manifoldcf.core.lockmanager.LockManager.enterLocks(LockManager.java:924)
at org.apache.manifoldcf.core.cachemanager.CacheManager.enterCache(CacheManager.java:278)
at org.apache.manifoldcf.core.cachemanager.CacheManager.findObjectsAndExecute(CacheManager.java:98)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.loadMultiple(OutputConnectionManager.java:256)
at org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.load(OutputConnectionManager.java:233)
at org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadOutputConnection(ManifoldCF.java:2122)
at org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3191)
at org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
at org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file lock 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock' failed: The system cannot find the path specified
java.io.IOException: The system cannot find the path specified
at java.io.WinNTFileSystem.createFileExclusively(Native Method)
at java.io.File.createNewFile(Unknown Source)
at org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
at org.apache.manifoldcf.core.lockmanager.LockObject.leaveReadLock(LockObject.java:489)
at org.apache.manifoldcf.core.lockmanager.LockManager.leaveReadLock(LockManager.java:752)
at org.apache.manifoldcf.core.lockmanager.LockManager.leaveLocks(LockManager.java:1216)
at org.apache.manifoldcf.core.cachemanager.CacheManager.commitTransaction(CacheManager.java:761)
at org.apache.manifoldcf.core.database.Database.endTransaction(Database.java:395)
at org.apache.manifoldcf.core.database.DBInterfacePostgreSQL.endTransaction(DBInterfacePostgreSQL.java:1161)
at org.apache.manifoldcf.core.database.BaseTable.endTransaction(BaseTable.java:277)
at org.apache.manifoldcf.crawler.jobs.Jobs.getAll(Jobs.java:538)
at org.apache.manifoldcf.crawler.jobs.JobManager.getAllJobs(JobManager.java:486)
at org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadJobs(ManifoldCF.java:1753)
at org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3069)
at org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
at org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
Does anyone have any ideas on what might be causing this? I note that although my AWS Ubuntu instance doesn't fail (or generate warnings), the syncharea does seem to get awfully large. Is it the intention that the syncharea should be self-cleansing?
Adrian
p.s.
I should note that my (home-grown) ManifoldCF controller software generates large amounts of ManifoldCF jobs, and is always tinkering with them (starting, stopping, modifying, etc.).
____________________________________________________________
Electronic mail messages entering and leaving Arup business
systems are scanned for acceptability of content and viruses
Re: Lock Warnings in ManifoldCF logs
Posted by Karl Wright <da...@gmail.com>.
Hi Adrian,
"WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file
lock
'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock'
failed: The system cannot find the path specified
java.io.IOException: The system cannot find the path specified
at java.io.WinNTFileSystem.
createFileExclusively(Native Method)"
This does not have anything to do with your controller software. What it
is complaining about is that it cannot create a file. There are a number
of possible reasons for this.
(1) You may be out of system resources
(2) You may have run ManifoldCF scripts as an incorrect user, causing
directories to be created which other manifoldcf processes don't have write
access to
(3) Windows may have gotten itself into a funky state
My suggestion is to first of all make SURE your ManifoldCF processes are
all being executed by the same user. THEN perform the lock clean sequence:
- shut down all manifoldcf processes
- run the lockclean.bat script
- start your manifoldcf processes again
Thanks,
Karl
On Wed, Jan 8, 2014 at 11:51 AM, Adrian Conlon <Ad...@arup.com>wrote:
> Hi All,
>
>
>
> I’m getting large numbers of locking warnings in my logs on a Windows
> server (normally I run under Ubuntu/Linux). Eventually the agent service
> stops working.
>
>
>
> Anyway, here’s a couple of sample exceptions from the log:
>
>
>
> WARN 2014-01-07 02:04:47,740 (http-bio-8080-exec-20) - Attempt to set file
> lock
> 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\32\542\lock-_Cache_OUTPUTCONNECTION_Oasys&58!
> Output connection.lock' failed: Access is denied
>
> java.io.IOException: Access is denied
>
> at java.io.WinNTFileSystem.createFileExclusively(Native
> Method)
>
> at java.io.File.createNewFile(Unknown Source)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLockNoWait(LockObject.java:439)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockObject.enterReadLock(LockObject.java:401)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockManager.enterLocks(LockManager.java:924)
>
> at
> org.apache.manifoldcf.core.cachemanager.CacheManager.enterCache(CacheManager.java:278)
>
> at
> org.apache.manifoldcf.core.cachemanager.CacheManager.findObjectsAndExecute(CacheManager.java:98)
>
> at
> org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.loadMultiple(OutputConnectionManager.java:256)
>
> at
> org.apache.manifoldcf.agents.outputconnection.OutputConnectionManager.load(OutputConnectionManager.java:233)
>
> at
> org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadOutputConnection(ManifoldCF.java:2122)
>
> at
> org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3191)
>
> at
> org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
>
> at
> org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
>
> at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
>
> at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
>
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
>
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
>
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
>
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
>
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
>
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
>
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
>
> at
> org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
>
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
>
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
>
> at
> org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
>
> at
> org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
>
> at
> org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
>
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
>
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
>
> at java.lang.Thread.run(Unknown Source)
>
>
>
> WARN 2014-01-06 11:06:00,782 (http-bio-8080-exec-5) - Attempt to set file
> lock
> 'C:\Solr\Application\crawler\Enterprise-Search\.\syncharea\392\308\lock-_Cache_JOB_1387380847667.lock'
> failed: The system cannot find the path specified
>
> java.io.IOException: The system cannot find the path specified
>
> at java.io.WinNTFileSystem.createFileExclusively(Native
> Method)
>
> at java.io.File.createNewFile(Unknown Source)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockObject.grabFileLock(LockObject.java:550)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockObject.leaveReadLock(LockObject.java:489)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockManager.leaveReadLock(LockManager.java:752)
>
> at
> org.apache.manifoldcf.core.lockmanager.LockManager.leaveLocks(LockManager.java:1216)
>
> at
> org.apache.manifoldcf.core.cachemanager.CacheManager.commitTransaction(CacheManager.java:761)
>
> at
> org.apache.manifoldcf.core.database.Database.endTransaction(Database.java:395)
>
> at
> org.apache.manifoldcf.core.database.DBInterfacePostgreSQL.endTransaction(DBInterfacePostgreSQL.java:1161)
>
> at
> org.apache.manifoldcf.core.database.BaseTable.endTransaction(BaseTable.java:277)
>
> at
> org.apache.manifoldcf.crawler.jobs.Jobs.getAll(Jobs.java:538)
>
> at
> org.apache.manifoldcf.crawler.jobs.JobManager.getAllJobs(JobManager.java:486)
>
> at
> org.apache.manifoldcf.crawler.system.ManifoldCF.apiReadJobs(ManifoldCF.java:1753)
>
> at
> org.apache.manifoldcf.crawler.system.ManifoldCF.executeReadCommand(ManifoldCF.java:3069)
>
> at
> org.apache.manifoldcf.apiservlet.APIServlet.executeRead(APIServlet.java:231)
>
> at
> org.apache.manifoldcf.apiservlet.APIServlet.doGet(APIServlet.java:77)
>
> at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:621)
>
> at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:722)
>
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
>
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
>
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225)
>
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
>
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
>
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
>
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
>
> at
> org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)
>
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
>
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
>
> at
> org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999)
>
> at
> org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565)
>
> at
> org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)
>
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
>
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
>
> at java.lang.Thread.run(Unknown Source)
>
>
>
>
>
> Does anyone have any ideas on what might be causing this? I note that
> although my AWS Ubuntu instance doesn’t fail (or generate warnings), the
> syncharea does seem to get awfully large. Is it the intention that the
> syncharea should be self-cleansing?
>
>
>
> Adrian
>
>
>
> p.s.
>
> I should note that my (home-grown) ManifoldCF controller software
> generates large amounts of ManifoldCF jobs, and is always tinkering with
> them (starting, stopping, modifying, etc.).
>
>
>
> ____________________________________________________________
> Electronic mail messages entering and leaving Arup business
> systems are scanned for acceptability of content and viruses
>