You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@camel.apache.org by Arjan Moraal <na...@ajmoraal.fastmail.net> on 2007/10/31 12:59:06 UTC
Problem with concurrent dir/file access?
Hi all,
I have a problem with the FileComponent endpoint. In our application, a lot
of files are written at about the same time to a directory that might not
exist yet. This often causes a FileNotFoundException. Or the directory is
created as file instead of directory.
I managed to extract it into a unit test that fails with a
FileNotFoundException on the directory (See bottom of this message).
I think the problem is in the FileProducer.process() method where it first
tries to build the filename String, and then creates the parent directories:
File file = createFileName(exchange.getIn());
buildDirectory(file);
However, the createFileName() method already checks whether the directory
exists (if (endpointFile.isDirectory()), which might not be the case since
buildDirectory() is called after createFileName()...
I don't have a solution/patch yet, but I thought I just let you know anyway
:)
Thanks,
Arjan
package com.bbc.newsi.feeds.sport.renderservice.camel;
import static org.apache.camel.language.juel.JuelExpression.el;
import java.io.File;
import junit.framework.TestCase;
import org.apache.camel.CamelContext;
import org.apache.camel.CamelTemplate;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.file.FileComponent;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
/**
*
* @author Albert Moraal
* @version $Revision$ $Date$ $Author$
*/
public class FileAsDirBugTest extends TestCase {
private static final Logger log =
Logger.getLogger(FileAsDirBugTest.class);
private CamelContext context;
private CamelTemplate template;
@Override
protected void setUp() throws Exception {
super.setUp();
BasicConfigurator.configure();
Logger.getRootLogger().setLevel(Level.DEBUG);
context = new DefaultCamelContext();
context.start();
template = new CamelTemplate(context);
}
@Override
protected void tearDown() throws Exception {
context.stop();
super.tearDown();
}
public void testFileCreatedAsDir() throws Exception {
log.debug("testFileCreatedAsDir");
final String path = "a/b/c/d/e/f/g/h";
final int numFiles = 10;
context.addRoutes(
new RouteBuilder() {
@Override
public void configure() {
String[] destinations = new String[numFiles];
for (int i=0; i<numFiles; i++) {
destinations[i] = "seda:file"+i;
from("seda:file"+i)
.setHeader(FileComponent.HEADER_FILE_NAME,
el("file"+i+".txt"))
.to("file://"+path+"/?append=false&noop=true");
}
from("seda:testFileCreatedAsDir")
.to(destinations);
}
}
);
FileUtils.deleteDirectory(new File("a"));
template.send("seda:testFileCreatedAsDir", new Processor() {
public void process(Exchange exchange) {
Message in = exchange.getIn();
in.setBody("Contents of test file");
}
});
Thread.sleep(2*1000);
for (int i=0; i<numFiles; i++) {
assertTrue((new File(path+"/file"+i+".txt")).isFile());
}
}
}
--
View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13507397
Sent from the Camel - Users mailing list archive at Nabble.com.
Re: Problem with concurrent dir/file access?
Posted by Arjan Moraal <na...@ajmoraal.fastmail.net>.
Here's a patch for FileProducer.java that fixes the "directory being created
as file" issue.
protected File createFileName(Message message) {
File answer;
File endpointFile = endpoint.getFile();
String name = null;
if (!endpoint.isIgnoreFileNameHeader()) {
name = message.getHeader(FileComponent.HEADER_FILE_NAME,
String.class);
}
if (endpointFile.isDirectory()) {
if (name != null) {
answer = new File(endpointFile, name);
if (answer.isDirectory()) {
answer = new File(answer, message.getMessageId());
}
} else {
answer = new File(endpointFile, message.getMessageId());
}
} else {
if (name == null) {
answer = endpointFile;
} else {
answer = new File(endpointFile, name);
}
}
return answer;
}
After applying this patch, the unit tests passes every time.
However, I still do get exceptions in the log output. It looks like a
multi-threading issue. As you can see from the logs below, the
buildDirectory() for file9 is entered while buildDirectory() for file 0 is
still running
500 [main] DEBUG org.apache.camel.util.ProducerCache - >>>>
Endpoint[seda:testFileCreatedAsDir] Exchange[Message: Contents of test file]
540 [seda:file0 thread:1] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file0.txt)
540 [seda:file9 thread:10] DEBUG
org.apache.camel.component.file.FileProducer -
buildDirectory(a\b\c\d\e\f\g\h\file9.txt)
540 [seda:file0 thread:1] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file0.txt) finished
570 [seda:file0 thread:1] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file0.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
570 [seda:file0 thread:1] ERROR org.apache.camel.processor.DeadLetterChannel
- On delivery attempt: 0 caught: java.io.FileNotFoundException:
a\b\c\d\e\f\g\h\file0.txt (The system cannot find the path specified)
java.io.FileNotFoundException: a\b\c\d\e\f\g\h\file0.txt (The system cannot
find the path specified)
at java.io.FileOutputStream.open(Native Method)
at java.io.FileOutputStream.<init>(FileOutputStream.java:179)
at java.io.FileOutputStream.<init>(FileOutputStream.java:131)
at
org.apache.camel.component.file.FileProducer.process(FileProducer.java:84)
at
org.apache.camel.component.file.FileProducer.process(FileProducer.java:60)
at
org.apache.camel.impl.converter.AsyncProcessorTypeConverter$ProcessorToAsynProcessorBridge.process(AsyncProcessorTypeConverter.java:44)
at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:73)
at
org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:136)
at
org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:86)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:85)
at
org.apache.camel.processor.UnitOfWorkProcessor.process(UnitOfWorkProcessor.java:53)
at org.apache.camel.component.seda.SedaConsumer.run(SedaConsumer.java:61)
at java.lang.Thread.run(Thread.java:595)
570 [seda:file0 thread:1] DEBUG org.apache.camel.processor.DeadLetterChannel
- Sleeping for: 1000 until attempting redelivery
580 [seda:file1 thread:2] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file1.txt)
580 [seda:file1 thread:2] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file1.txt) finished
580 [seda:file1 thread:2] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file1.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
590 [seda:file2 thread:3] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file2.txt)
590 [seda:file2 thread:3] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file2.txt) finished
590 [seda:file3 thread:4] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file3.txt)
590 [seda:file4 thread:5] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file4.txt)
590 [seda:file5 thread:6] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file5.txt)
590 [seda:file6 thread:7] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file6.txt)
590 [seda:file7 thread:8] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file7.txt)
590 [seda:file9 thread:10] DEBUG
org.apache.camel.component.file.FileProducer -
buildDirectory(a\b\c\d\e\f\g\h\file9.txt) finished
590 [seda:file8 thread:9] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file8.txt)
590 [seda:file2 thread:3] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file2.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
590 [seda:file3 thread:4] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file3.txt) finished
590 [seda:file4 thread:5] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file4.txt) finished
590 [seda:file5 thread:6] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file5.txt) finished
590 [seda:file6 thread:7] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file6.txt) finished
590 [seda:file7 thread:8] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file7.txt) finished
590 [seda:file9 thread:10] DEBUG
org.apache.camel.component.file.FileProducer - About to write to:
a\b\c\d\e\f\g\h\file9.txt from exchange: Exchange[FileMessage:
a\b\c\d\e\f\g\h]
590 [seda:file8 thread:9] DEBUG org.apache.camel.component.file.FileProducer
- buildDirectory(a\b\c\d\e\f\g\h\file8.txt) finished
590 [seda:file4 thread:5] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file4.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
590 [seda:file5 thread:6] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file5.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
590 [seda:file6 thread:7] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file6.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
590 [seda:file3 thread:4] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file3.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
590 [seda:file7 thread:8] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file7.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
600 [seda:file8 thread:9] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h\file8.txt from exchange:
Exchange[FileMessage: a\b\c\d\e\f\g\h]
1572 [seda:file0 thread:1] DEBUG
org.apache.camel.component.file.FileProducer -
buildDirectory(a\b\c\d\e\f\g\h\file0.txt)
1572 [seda:file0 thread:1] DEBUG
org.apache.camel.component.file.FileProducer -
buildDirectory(a\b\c\d\e\f\g\h\file0.txt) finished
1572 [seda:file0 thread:1] DEBUG
org.apache.camel.component.file.FileProducer - About to write to:
a\b\c\d\e\f\g\h\file0.txt from exchange: Exchange[FileMessage:
a\b\c\d\e\f\g\h]
--
View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13543906
Sent from the Camel - Users mailing list archive at Nabble.com.
Re: Problem with concurrent dir/file access?
Posted by Arjan Moraal <na...@ajmoraal.fastmail.net>.
I've added a patch for this issue to Jira:
https://issues.apache.org/activemq/browse/CAMEL-203
Could someone please have a look and add it to the trunk?
Thanks,
Arjan
--
View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13603541
Sent from the Camel - Users mailing list archive at Nabble.com.
Re: Problem with concurrent dir/file access?
Posted by Arjan Moraal <na...@ajmoraal.fastmail.net>.
On running it a couple of times I noticed it sometimes passes, sometimes
fails.
When changing the path to a shorter structure, like "a/b/" it passes all the
time, but a longer structure like "a/b/c/d/e/f/g/h/" often fails with the
last element being created as file instead of directory.
Arjan Moraal wrote:
>
> Thanks James.
> I ran the test case from the trunk now. The first time it passed, then I
> deleted the "a" directory, ran the test again, then I got the following
> Exception:
>
> 561 [seda:file1 thread:2] DEBUG
> org.apache.camel.component.file.FileProducer - About to write to:
> a\b\c\d\e\f\g\h from exchange: Exchange[FileMessage: a\b\c\d\e\f\g\h]
> 561 [seda:file1 thread:2] ERROR
> org.apache.camel.processor.DeadLetterChannel - On delivery attempt: 0
> caught: java.io.FileNotFoundException: a\b\c\d\e\f\g\h (The system cannot
> find the path specified)
> java.io.FileNotFoundException: a\b\c\d\e\f\g\h (The system cannot find the
> path specified)
> at java.io.FileOutputStream.open(Native Method)
> at java.io.FileOutputStream.<init>(FileOutputStream.java:179)
> at java.io.FileOutputStream.<init>(FileOutputStream.java:131)
> at
> org.apache.camel.component.file.FileProducer.process(FileProducer.java:84)
> at
> org.apache.camel.component.file.FileProducer.process(FileProducer.java:60)
> at
> org.apache.camel.impl.converter.AsyncProcessorTypeConverter$ProcessorToAsynProcessorBridge.process(AsyncProcessorTypeConverter.java:44)
> at
> org.apache.camel.processor.SendProcessor.process(SendProcessor.java:73)
> at
> org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:136)
> at
> org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:86)
> at org.apache.camel.processor.Pipeline.process(Pipeline.java:101)
> at org.apache.camel.processor.Pipeline.process(Pipeline.java:85)
> at
> org.apache.camel.processor.UnitOfWorkProcessor.process(UnitOfWorkProcessor.java:53)
> at org.apache.camel.component.seda.SedaConsumer.run(SedaConsumer.java:61)
> at java.lang.Thread.run(Thread.java:595)
>
> Looking at the directory structure, it created "h" as file instead of
> directory.
>
> Arjan
>
>
--
View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13511880
Sent from the Camel - Users mailing list archive at Nabble.com.
Re: Problem with concurrent dir/file access?
Posted by Arjan Moraal <na...@ajmoraal.fastmail.net>.
Thanks James.
I ran the test case from the trunk now. The first time it passed, then I
deleted the "a" directory, ran the test again, then I got the following
Exception:
561 [seda:file1 thread:2] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h from exchange: Exchange[FileMessage:
a\b\c\d\e\f\g\h]
561 [seda:file1 thread:2] ERROR org.apache.camel.processor.DeadLetterChannel
- On delivery attempt: 0 caught: java.io.FileNotFoundException:
a\b\c\d\e\f\g\h (The system cannot find the path specified)
java.io.FileNotFoundException: a\b\c\d\e\f\g\h (The system cannot find the
path specified)
at java.io.FileOutputStream.open(Native Method)
at java.io.FileOutputStream.<init>(FileOutputStream.java:179)
at java.io.FileOutputStream.<init>(FileOutputStream.java:131)
at
org.apache.camel.component.file.FileProducer.process(FileProducer.java:84)
at
org.apache.camel.component.file.FileProducer.process(FileProducer.java:60)
at
org.apache.camel.impl.converter.AsyncProcessorTypeConverter$ProcessorToAsynProcessorBridge.process(AsyncProcessorTypeConverter.java:44)
at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:73)
at
org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:136)
at
org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:86)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:85)
at
org.apache.camel.processor.UnitOfWorkProcessor.process(UnitOfWorkProcessor.java:53)
at org.apache.camel.component.seda.SedaConsumer.run(SedaConsumer.java:61)
at java.lang.Thread.run(Thread.java:595)
Looking at the directory structure, it created "h" as file instead of
directory.
Arjan
James.Strachan wrote:
>
> Thanks for the heads up Arjan! I've added the test case to the trunk -
> though it seems to work fine for me :(
>
> See DirectoryCreateIssueTest in the camel-core module.
>
> (I tried removing the dependency on Juel and commons-io so that the
> test could easily live in the camel-core test library). Maybe I broke
> something during this refactor?
>
>
> On 31/10/2007, Arjan Moraal <na...@ajmoraal.fastmail.net> wrote:
>>
>> Hi all,
>>
>> I have a problem with the FileComponent endpoint. In our application, a
>> lot
>> of files are written at about the same time to a directory that might not
>> exist yet. This often causes a FileNotFoundException. Or the directory is
>> created as file instead of directory.
>>
>> I managed to extract it into a unit test that fails with a
>> FileNotFoundException on the directory (See bottom of this message).
>>
>> I think the problem is in the FileProducer.process() method where it
>> first
>> tries to build the filename String, and then creates the parent
>> directories:
>> File file = createFileName(exchange.getIn());
>> buildDirectory(file);
>> However, the createFileName() method already checks whether the directory
>> exists (if (endpointFile.isDirectory()), which might not be the case
>> since
>> buildDirectory() is called after createFileName()...
>>
>> I don't have a solution/patch yet, but I thought I just let you know
>> anyway
>> :)
>>
>> Thanks,
>> Arjan
>>
>> package com.bbc.newsi.feeds.sport.renderservice.camel;
>>
>> import static org.apache.camel.language.juel.JuelExpression.el;
>>
>> import java.io.File;
>>
>> import junit.framework.TestCase;
>>
>> import org.apache.camel.CamelContext;
>> import org.apache.camel.CamelTemplate;
>> import org.apache.camel.Exchange;
>> import org.apache.camel.Message;
>> import org.apache.camel.Processor;
>> import org.apache.camel.builder.RouteBuilder;
>> import org.apache.camel.component.file.FileComponent;
>> import org.apache.camel.impl.DefaultCamelContext;
>> import org.apache.commons.io.FileUtils;
>> import org.apache.log4j.BasicConfigurator;
>> import org.apache.log4j.Level;
>> import org.apache.log4j.Logger;
>>
>> /**
>> *
>> * @author Albert Moraal
>> * @version $Revision$ $Date$ $Author$
>> */
>> public class FileAsDirBugTest extends TestCase {
>> private static final Logger log =
>> Logger.getLogger(FileAsDirBugTest.class);
>>
>> private CamelContext context;
>> private CamelTemplate template;
>>
>> @Override
>> protected void setUp() throws Exception {
>> super.setUp();
>> BasicConfigurator.configure();
>> Logger.getRootLogger().setLevel(Level.DEBUG);
>> context = new DefaultCamelContext();
>> context.start();
>> template = new CamelTemplate(context);
>> }
>>
>> @Override
>> protected void tearDown() throws Exception {
>> context.stop();
>> super.tearDown();
>> }
>>
>> public void testFileCreatedAsDir() throws Exception {
>> log.debug("testFileCreatedAsDir");
>> final String path = "a/b/c/d/e/f/g/h";
>> final int numFiles = 10;
>>
>> context.addRoutes(
>> new RouteBuilder() {
>> @Override
>> public void configure() {
>> String[] destinations = new String[numFiles];
>> for (int i=0; i<numFiles; i++) {
>> destinations[i] = "seda:file"+i;
>>
>> from("seda:file"+i)
>> .setHeader(FileComponent.HEADER_FILE_NAME,
>> el("file"+i+".txt"))
>>
>> .to("file://"+path+"/?append=false&noop=true");
>> }
>>
>> from("seda:testFileCreatedAsDir")
>> .to(destinations);
>> }
>> }
>> );
>>
>> FileUtils.deleteDirectory(new File("a"));
>>
>> template.send("seda:testFileCreatedAsDir", new Processor() {
>> public void process(Exchange exchange) {
>> Message in = exchange.getIn();
>> in.setBody("Contents of test file");
>> }
>> });
>>
>> Thread.sleep(2*1000);
>> for (int i=0; i<numFiles; i++) {
>> assertTrue((new File(path+"/file"+i+".txt")).isFile());
>> }
>> }
>>
>> }
>>
>>
>> --
>> View this message in context:
>> http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13507397
>> Sent from the Camel - Users mailing list archive at Nabble.com.
>>
>>
>
>
> --
> James
> -------
> http://macstrac.blogspot.com/
>
> Open Source SOA
> http://open.iona.com
>
>
--
View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13511448
Sent from the Camel - Users mailing list archive at Nabble.com.
Re: Problem with concurrent dir/file access?
Posted by Arjan Moraal <na...@ajmoraal.fastmail.net>.
Thanks James.
I ran the test case from the trunk now. The first time it passed, then I
deleted the "a" directory, ran the test again, then I got the following
Exception:
561 [seda:file1 thread:2] DEBUG org.apache.camel.component.file.FileProducer
- About to write to: a\b\c\d\e\f\g\h from exchange: Exchange[FileMessage:
a\b\c\d\e\f\g\h]
561 [seda:file1 thread:2] ERROR org.apache.camel.processor.DeadLetterChannel
- On delivery attempt: 0 caught: java.io.FileNotFoundException:
a\b\c\d\e\f\g\h (The system cannot find the path specified)
java.io.FileNotFoundException: a\b\c\d\e\f\g\h (The system cannot find the
path specified)
at java.io.FileOutputStream.open(Native Method)
at java.io.FileOutputStream.<init>(FileOutputStream.java:179)
at java.io.FileOutputStream.<init>(FileOutputStream.java:131)
at
org.apache.camel.component.file.FileProducer.process(FileProducer.java:84)
at
org.apache.camel.component.file.FileProducer.process(FileProducer.java:60)
at
org.apache.camel.impl.converter.AsyncProcessorTypeConverter$ProcessorToAsynProcessorBridge.process(AsyncProcessorTypeConverter.java:44)
at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:73)
at
org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:136)
at
org.apache.camel.processor.DeadLetterChannel.process(DeadLetterChannel.java:86)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:101)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:85)
at
org.apache.camel.processor.UnitOfWorkProcessor.process(UnitOfWorkProcessor.java:53)
at org.apache.camel.component.seda.SedaConsumer.run(SedaConsumer.java:61)
at java.lang.Thread.run(Thread.java:595)
Looking at the directory structure, it created "h" as file instead of
directory.
Albert
James.Strachan wrote:
>
> Thanks for the heads up Arjan! I've added the test case to the trunk -
> though it seems to work fine for me :(
>
> See DirectoryCreateIssueTest in the camel-core module.
>
> (I tried removing the dependency on Juel and commons-io so that the
> test could easily live in the camel-core test library). Maybe I broke
> something during this refactor?
>
>
> On 31/10/2007, Arjan Moraal <na...@ajmoraal.fastmail.net> wrote:
>>
>> Hi all,
>>
>> I have a problem with the FileComponent endpoint. In our application, a
>> lot
>> of files are written at about the same time to a directory that might not
>> exist yet. This often causes a FileNotFoundException. Or the directory is
>> created as file instead of directory.
>>
>> I managed to extract it into a unit test that fails with a
>> FileNotFoundException on the directory (See bottom of this message).
>>
>> I think the problem is in the FileProducer.process() method where it
>> first
>> tries to build the filename String, and then creates the parent
>> directories:
>> File file = createFileName(exchange.getIn());
>> buildDirectory(file);
>> However, the createFileName() method already checks whether the directory
>> exists (if (endpointFile.isDirectory()), which might not be the case
>> since
>> buildDirectory() is called after createFileName()...
>>
>> I don't have a solution/patch yet, but I thought I just let you know
>> anyway
>> :)
>>
>> Thanks,
>> Arjan
>>
>> package com.bbc.newsi.feeds.sport.renderservice.camel;
>>
>> import static org.apache.camel.language.juel.JuelExpression.el;
>>
>> import java.io.File;
>>
>> import junit.framework.TestCase;
>>
>> import org.apache.camel.CamelContext;
>> import org.apache.camel.CamelTemplate;
>> import org.apache.camel.Exchange;
>> import org.apache.camel.Message;
>> import org.apache.camel.Processor;
>> import org.apache.camel.builder.RouteBuilder;
>> import org.apache.camel.component.file.FileComponent;
>> import org.apache.camel.impl.DefaultCamelContext;
>> import org.apache.commons.io.FileUtils;
>> import org.apache.log4j.BasicConfigurator;
>> import org.apache.log4j.Level;
>> import org.apache.log4j.Logger;
>>
>> /**
>> *
>> * @author Albert Moraal
>> * @version $Revision$ $Date$ $Author$
>> */
>> public class FileAsDirBugTest extends TestCase {
>> private static final Logger log =
>> Logger.getLogger(FileAsDirBugTest.class);
>>
>> private CamelContext context;
>> private CamelTemplate template;
>>
>> @Override
>> protected void setUp() throws Exception {
>> super.setUp();
>> BasicConfigurator.configure();
>> Logger.getRootLogger().setLevel(Level.DEBUG);
>> context = new DefaultCamelContext();
>> context.start();
>> template = new CamelTemplate(context);
>> }
>>
>> @Override
>> protected void tearDown() throws Exception {
>> context.stop();
>> super.tearDown();
>> }
>>
>> public void testFileCreatedAsDir() throws Exception {
>> log.debug("testFileCreatedAsDir");
>> final String path = "a/b/c/d/e/f/g/h";
>> final int numFiles = 10;
>>
>> context.addRoutes(
>> new RouteBuilder() {
>> @Override
>> public void configure() {
>> String[] destinations = new String[numFiles];
>> for (int i=0; i<numFiles; i++) {
>> destinations[i] = "seda:file"+i;
>>
>> from("seda:file"+i)
>> .setHeader(FileComponent.HEADER_FILE_NAME,
>> el("file"+i+".txt"))
>>
>> .to("file://"+path+"/?append=false&noop=true");
>> }
>>
>> from("seda:testFileCreatedAsDir")
>> .to(destinations);
>> }
>> }
>> );
>>
>> FileUtils.deleteDirectory(new File("a"));
>>
>> template.send("seda:testFileCreatedAsDir", new Processor() {
>> public void process(Exchange exchange) {
>> Message in = exchange.getIn();
>> in.setBody("Contents of test file");
>> }
>> });
>>
>> Thread.sleep(2*1000);
>> for (int i=0; i<numFiles; i++) {
>> assertTrue((new File(path+"/file"+i+".txt")).isFile());
>> }
>> }
>>
>> }
>>
>>
>> --
>> View this message in context:
>> http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13507397
>> Sent from the Camel - Users mailing list archive at Nabble.com.
>>
>>
>
>
> --
> James
> -------
> http://macstrac.blogspot.com/
>
> Open Source SOA
> http://open.iona.com
>
>
--
View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13511448
Sent from the Camel - Users mailing list archive at Nabble.com.
Re: Problem with concurrent dir/file access?
Posted by James Strachan <ja...@gmail.com>.
Thanks for the heads up Arjan! I've added the test case to the trunk -
though it seems to work fine for me :(
See DirectoryCreateIssueTest in the camel-core module.
(I tried removing the dependency on Juel and commons-io so that the
test could easily live in the camel-core test library). Maybe I broke
something during this refactor?
On 31/10/2007, Arjan Moraal <na...@ajmoraal.fastmail.net> wrote:
>
> Hi all,
>
> I have a problem with the FileComponent endpoint. In our application, a lot
> of files are written at about the same time to a directory that might not
> exist yet. This often causes a FileNotFoundException. Or the directory is
> created as file instead of directory.
>
> I managed to extract it into a unit test that fails with a
> FileNotFoundException on the directory (See bottom of this message).
>
> I think the problem is in the FileProducer.process() method where it first
> tries to build the filename String, and then creates the parent directories:
> File file = createFileName(exchange.getIn());
> buildDirectory(file);
> However, the createFileName() method already checks whether the directory
> exists (if (endpointFile.isDirectory()), which might not be the case since
> buildDirectory() is called after createFileName()...
>
> I don't have a solution/patch yet, but I thought I just let you know anyway
> :)
>
> Thanks,
> Arjan
>
> package com.bbc.newsi.feeds.sport.renderservice.camel;
>
> import static org.apache.camel.language.juel.JuelExpression.el;
>
> import java.io.File;
>
> import junit.framework.TestCase;
>
> import org.apache.camel.CamelContext;
> import org.apache.camel.CamelTemplate;
> import org.apache.camel.Exchange;
> import org.apache.camel.Message;
> import org.apache.camel.Processor;
> import org.apache.camel.builder.RouteBuilder;
> import org.apache.camel.component.file.FileComponent;
> import org.apache.camel.impl.DefaultCamelContext;
> import org.apache.commons.io.FileUtils;
> import org.apache.log4j.BasicConfigurator;
> import org.apache.log4j.Level;
> import org.apache.log4j.Logger;
>
> /**
> *
> * @author Albert Moraal
> * @version $Revision$ $Date$ $Author$
> */
> public class FileAsDirBugTest extends TestCase {
> private static final Logger log =
> Logger.getLogger(FileAsDirBugTest.class);
>
> private CamelContext context;
> private CamelTemplate template;
>
> @Override
> protected void setUp() throws Exception {
> super.setUp();
> BasicConfigurator.configure();
> Logger.getRootLogger().setLevel(Level.DEBUG);
> context = new DefaultCamelContext();
> context.start();
> template = new CamelTemplate(context);
> }
>
> @Override
> protected void tearDown() throws Exception {
> context.stop();
> super.tearDown();
> }
>
> public void testFileCreatedAsDir() throws Exception {
> log.debug("testFileCreatedAsDir");
> final String path = "a/b/c/d/e/f/g/h";
> final int numFiles = 10;
>
> context.addRoutes(
> new RouteBuilder() {
> @Override
> public void configure() {
> String[] destinations = new String[numFiles];
> for (int i=0; i<numFiles; i++) {
> destinations[i] = "seda:file"+i;
>
> from("seda:file"+i)
> .setHeader(FileComponent.HEADER_FILE_NAME,
> el("file"+i+".txt"))
> .to("file://"+path+"/?append=false&noop=true");
> }
>
> from("seda:testFileCreatedAsDir")
> .to(destinations);
> }
> }
> );
>
> FileUtils.deleteDirectory(new File("a"));
>
> template.send("seda:testFileCreatedAsDir", new Processor() {
> public void process(Exchange exchange) {
> Message in = exchange.getIn();
> in.setBody("Contents of test file");
> }
> });
>
> Thread.sleep(2*1000);
> for (int i=0; i<numFiles; i++) {
> assertTrue((new File(path+"/file"+i+".txt")).isFile());
> }
> }
>
> }
>
>
> --
> View this message in context: http://www.nabble.com/Problem-with-concurrent-dir-file-access--tf4724448s22882.html#a13507397
> Sent from the Camel - Users mailing list archive at Nabble.com.
>
>
--
James
-------
http://macstrac.blogspot.com/
Open Source SOA
http://open.iona.com