You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2016/01/30 09:43:39 UTC
camel git commit: CAMEL-9550: File consumer - Use Filter while
deleting orphaned marker lock files
Repository: camel
Updated Branches:
refs/heads/master 83151a1f2 -> c8430ab5a
CAMEL-9550: File consumer - Use Filter while deleting orphaned marker lock files
Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/c8430ab5
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/c8430ab5
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/c8430ab5
Branch: refs/heads/master
Commit: c8430ab5a1f442c8d1ded61811522081dd8723d0
Parents: 83151a1
Author: Claus Ibsen <da...@apache.org>
Authored: Sat Jan 30 09:42:24 2016 +0100
Committer: Claus Ibsen <da...@apache.org>
Committed: Sat Jan 30 09:42:24 2016 +0100
----------------------------------------------------------------------
.../MarkerFileExclusiveReadLockStrategy.java | 91 +++++++++++++++++++-
...leRecursiveFilterDeleteOldLockFilesTest.java | 88 +++++++++++++++++++
2 files changed, 175 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/camel/blob/c8430ab5/camel-core/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
----------------------------------------------------------------------
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java b/camel-core/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
index 1c92bbd..0677dbf 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
@@ -17,6 +17,7 @@
package org.apache.camel.component.file.strategy;
import java.io.File;
+import java.util.regex.Pattern;
import org.apache.camel.Exchange;
import org.apache.camel.LoggingLevel;
@@ -24,8 +25,10 @@ import org.apache.camel.component.file.FileComponent;
import org.apache.camel.component.file.GenericFile;
import org.apache.camel.component.file.GenericFileEndpoint;
import org.apache.camel.component.file.GenericFileExclusiveReadLockStrategy;
+import org.apache.camel.component.file.GenericFileFilter;
import org.apache.camel.component.file.GenericFileOperations;
import org.apache.camel.util.FileUtil;
+import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.StopWatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,8 +52,12 @@ public class MarkerFileExclusiveReadLockStrategy implements GenericFileExclusive
LOG.debug("Prepare on startup by deleting orphaned lock files from: {}", dir);
+ Pattern excludePattern = endpoint.getExclude() != null ? Pattern.compile(endpoint.getExclude()) : null;
+ Pattern includePattern = endpoint.getInclude() != null ? Pattern.compile(endpoint.getInclude()) : null;
+ String endpointPath = endpoint.getConfiguration().getDirectory();
+
StopWatch watch = new StopWatch();
- deleteLockFiles(file, endpoint.isRecursive());
+ deleteLockFiles(file, endpoint.isRecursive(), endpointPath, endpoint.getFilter(), endpoint.getAntFilter(), excludePattern, includePattern);
// log anything that takes more than a second
if (watch.taken() > 1000) {
@@ -143,23 +150,99 @@ public class MarkerFileExclusiveReadLockStrategy implements GenericFileExclusive
this.deleteOrphanLockFiles = deleteOrphanLockFiles;
}
- private static void deleteLockFiles(File dir, boolean recursive) {
+ private static void deleteLockFiles(File dir, boolean recursive, String endpointPath,
+ GenericFileFilter filter, GenericFileFilter antFilter,
+ Pattern excludePattern, Pattern includePattern) {
File[] files = dir.listFiles();
if (files == null || files.length == 0) {
return;
}
for (File file : files) {
+
if (file.getName().startsWith(".")) {
// files starting with dot should be skipped
continue;
- } else if (file.getName().endsWith(FileComponent.DEFAULT_LOCK_FILE_POSTFIX)) {
+ }
+
+ // filter unwanted files and directories to avoid traveling everything
+ if (filter != null || antFilter != null || excludePattern != null || includePattern != null) {
+ if (!acceptFile(file, endpointPath, filter, antFilter, excludePattern, includePattern)) {
+ continue;
+ }
+ }
+
+ if (file.getName().endsWith(FileComponent.DEFAULT_LOCK_FILE_POSTFIX)) {
LOG.warn("Deleting orphaned lock file: " + file);
FileUtil.deleteFile(file);
} else if (recursive && file.isDirectory()) {
- deleteLockFiles(file, true);
+ deleteLockFiles(file, true, endpointPath, filter, antFilter, excludePattern, includePattern);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static boolean acceptFile(File file, String endpointPath, GenericFileFilter filter, GenericFileFilter antFilter,
+ Pattern excludePattern, Pattern includePattern) {
+ GenericFile gf = new GenericFile();
+ gf.setEndpointPath(endpointPath);
+ gf.setFile(file);
+ gf.setFileNameOnly(file.getName());
+ gf.setFileLength(file.length());
+ gf.setDirectory(file.isDirectory());
+ // must use FileUtil.isAbsolute to have consistent check for whether the file is
+ // absolute or not. As windows do not consider \ paths as absolute where as all
+ // other OS platforms will consider \ as absolute. The logic in Camel mandates
+ // that we align this for all OS. That is why we must use FileUtil.isAbsolute
+ // to return a consistent answer for all OS platforms.
+ gf.setAbsolute(FileUtil.isAbsolute(file));
+ gf.setAbsoluteFilePath(file.getAbsolutePath());
+ gf.setLastModified(file.lastModified());
+
+ // compute the file path as relative to the starting directory
+ File path;
+ String endpointNormalized = FileUtil.normalizePath(endpointPath);
+ if (file.getPath().startsWith(endpointNormalized + File.separator)) {
+ // skip duplicate endpoint path
+ path = new File(ObjectHelper.after(file.getPath(), endpointNormalized + File.separator));
+ } else {
+ path = new File(file.getPath());
+ }
+
+ if (path.getParent() != null) {
+ gf.setRelativeFilePath(path.getParent() + File.separator + file.getName());
+ } else {
+ gf.setRelativeFilePath(path.getName());
+ }
+
+ // the file name should be the relative path
+ gf.setFileName(gf.getRelativeFilePath());
+
+ if (filter != null) {
+ if (!filter.accept(gf)) {
+ return false;
+ }
+ }
+
+ if (antFilter != null) {
+ if (!antFilter.accept(gf)) {
+ return false;
}
}
+
+ // exclude take precedence over include
+ if (excludePattern != null) {
+ if (excludePattern.matcher(file.getName()).matches()) {
+ return false;
+ }
+ }
+ if (includePattern != null) {
+ if (!includePattern.matcher(file.getName()).matches()) {
+ return false;
+ }
+ }
+
+ return true;
}
private static String getLockFileName(GenericFile<File> file) {
http://git-wip-us.apache.org/repos/asf/camel/blob/c8430ab5/camel-core/src/test/java/org/apache/camel/component/file/FileMarkerFileRecursiveFilterDeleteOldLockFilesTest.java
----------------------------------------------------------------------
diff --git a/camel-core/src/test/java/org/apache/camel/component/file/FileMarkerFileRecursiveFilterDeleteOldLockFilesTest.java b/camel-core/src/test/java/org/apache/camel/component/file/FileMarkerFileRecursiveFilterDeleteOldLockFilesTest.java
new file mode 100644
index 0000000..cd20b3e
--- /dev/null
+++ b/camel-core/src/test/java/org/apache/camel/component/file/FileMarkerFileRecursiveFilterDeleteOldLockFilesTest.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file;
+
+import java.io.File;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.impl.JndiRegistry;
+
+/**
+ * @version
+ */
+public class FileMarkerFileRecursiveFilterDeleteOldLockFilesTest extends ContextTestSupport {
+
+ @Override
+ protected JndiRegistry createRegistry() throws Exception {
+ JndiRegistry jndi = super.createRegistry();
+ jndi.bind("myFilter", new MyFileFilter());
+ return jndi;
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ deleteDirectory("target/oldlock");
+ template.sendBodyAndHeader("file:target/oldlock", "locked", Exchange.FILE_NAME, "hello.txt" + FileComponent.DEFAULT_LOCK_FILE_POSTFIX);
+ template.sendBodyAndHeader("file:target/oldlock", "Bye World", Exchange.FILE_NAME, "bye.txt");
+ template.sendBodyAndHeader("file:target/oldlock/foo", "locked", Exchange.FILE_NAME, "gooday.txt" + FileComponent.DEFAULT_LOCK_FILE_POSTFIX);
+ template.sendBodyAndHeader("file:target/oldlock/foo", "Hi World", Exchange.FILE_NAME, "hi.txt");
+ template.sendBodyAndHeader("file:target/oldlock/bar", "locked", Exchange.FILE_NAME, "davs.txt" + FileComponent.DEFAULT_LOCK_FILE_POSTFIX);
+ template.sendBodyAndHeader("file:target/oldlock/bar", "Davs World", Exchange.FILE_NAME, "davs.txt");
+ }
+
+ public void testDeleteOldLockOnStartup() throws Exception {
+ MockEndpoint mock = getMockEndpoint("mock:result");
+ mock.expectedMessageCount(2);
+ mock.expectedBodiesReceived("Bye World", "Hi World");
+ mock.message(0).header(Exchange.FILE_NAME_ONLY).isEqualTo("bye.txt");
+ mock.message(1).header(Exchange.FILE_NAME_ONLY).isEqualTo("hi.txt");
+ mock.expectedFileExists("target/oldlock/bar/davs.txt");
+ mock.expectedFileExists("target/oldlock/bar/davs.txt" + FileComponent.DEFAULT_LOCK_FILE_POSTFIX);
+
+ // start the route
+ context.startRoute("foo");
+
+ assertMockEndpointsSatisfied();
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("file:target/oldlock?recursive=true&sortBy=file:name&filter=#myFilter").routeId("foo").noAutoStartup()
+ .convertBodyTo(String.class).to("mock:result");
+ }
+ };
+ }
+
+ private static final class MyFileFilter implements GenericFileFilter<File> {
+
+ @Override
+ public boolean accept(GenericFile<File> file) {
+ // skip bar directory
+ if (file.isDirectory() && "bar".equals(file.getFileName())) {
+ return false;
+ }
+ return true;
+ }
+ }
+}
\ No newline at end of file