You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2024/03/02 06:39:09 UTC
(camel) branch camel-4.4.x updated: CAMEL-20482: use min/max depth during markerfile cleanup (#13366)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch camel-4.4.x
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/camel-4.4.x by this push:
new a3a3501b9ee CAMEL-20482: use min/max depth during markerfile cleanup (#13366)
a3a3501b9ee is described below
commit a3a3501b9ee9a2351ada9fdd0cd7e4a675cfe710
Author: Stephen Higgs <sj...@users.noreply.github.com>
AuthorDate: Sat Mar 2 01:37:15 2024 -0500
CAMEL-20482: use min/max depth during markerfile cleanup (#13366)
* CAMEL-20482: use min/max depth during markerfile cleanup
* CAMEL-20482: modify cleanup logging to trace
---
.../MarkerFileExclusiveReadLockStrategy.java | 19 +-
...lusiveReadLockStrategyRecursiveCleanupTest.java | 199 +++++++++++++++++++++
2 files changed, 213 insertions(+), 5 deletions(-)
diff --git a/components/camel-file/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java b/components/camel-file/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
index 06cb6c47d49..82650398f48 100644
--- a/components/camel-file/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
+++ b/components/camel-file/src/main/java/org/apache/camel/component/file/strategy/MarkerFileExclusiveReadLockStrategy.java
@@ -57,7 +57,8 @@ public class MarkerFileExclusiveReadLockStrategy implements GenericFileExclusive
String endpointPath = endpoint.getConfiguration().getDirectory();
StopWatch watch = new StopWatch();
- deleteLockFiles(file, endpoint.isRecursive(), endpoint.isHiddenFilesEnabled(), endpointPath, endpoint.getFilter(),
+ deleteLockFiles(file, endpoint.isRecursive(), endpoint.getMinDepth(), endpoint.getMaxDepth(), 1,
+ endpoint.isHiddenFilesEnabled(), endpointPath, endpoint.getFilter(),
endpoint.getAntFilter(),
excludePattern, includePattern);
@@ -170,10 +171,16 @@ public class MarkerFileExclusiveReadLockStrategy implements GenericFileExclusive
}
private static <T> void deleteLockFiles(
- File dir, boolean recursive, boolean hiddenFilesEnabled, String endpointPath, GenericFileFilter<T> filter,
+ File dir, boolean recursive, int minDepth, int maxDepth, int depth, boolean hiddenFilesEnabled, String endpointPath,
+ GenericFileFilter<T> filter,
GenericFileFilter<T> antFilter,
Pattern excludePattern,
Pattern includePattern) {
+
+ if (recursive) {
+ LOG.trace("checking: depth {}, minDepth {}, maxDepth {}, directory: {}", depth, minDepth, maxDepth, dir);
+ }
+
File[] files = dir.listFiles();
if (files == null || files.length == 0) {
return;
@@ -210,11 +217,13 @@ public class MarkerFileExclusiveReadLockStrategy implements GenericFileExclusive
}
}
- if (file.getName().endsWith(FileComponent.DEFAULT_LOCK_FILE_POSTFIX)) {
+ if (file.getName().endsWith(FileComponent.DEFAULT_LOCK_FILE_POSTFIX) && (depth >= minDepth)) {
LOG.warn("Deleting orphaned lock file: {}", file);
FileUtil.deleteFile(file);
- } else if (recursive && file.isDirectory()) {
- deleteLockFiles(file, true, hiddenFilesEnabled, endpointPath, filter, antFilter, excludePattern,
+ } else if (recursive && file.isDirectory() && (depth < maxDepth)) {
+ deleteLockFiles(file, true, minDepth, maxDepth, depth + 1, hiddenFilesEnabled, endpointPath, filter,
+ antFilter,
+ excludePattern,
includePattern);
}
}
diff --git a/core/camel-core/src/test/java/org/apache/camel/component/file/MarkerFileExclusiveReadLockStrategyRecursiveCleanupTest.java b/core/camel-core/src/test/java/org/apache/camel/component/file/MarkerFileExclusiveReadLockStrategyRecursiveCleanupTest.java
new file mode 100644
index 00000000000..9be63945b0d
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/component/file/MarkerFileExclusiveReadLockStrategyRecursiveCleanupTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.builder.NotifyBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class MarkerFileExclusiveReadLockStrategyRecursiveCleanupTest extends ContextTestSupport {
+
+ public boolean isUseRouteBuilder() {
+ return false;
+ }
+
+ @Override
+ @BeforeEach
+ public void setUp() throws Exception {
+ super.setUp();
+
+ testDirectory("d1/d2/d3/d4/d5", true);
+
+ createFiles("d1", "d1.dat");
+ createFiles("d1/d2", "d2.dat");
+ createFiles("d1/d2/d3", "d3.dat");
+ createFiles("d1/d2/d3/d4", "d4.dat");
+ createFiles("d1/d2/d3/d4/d5", "d5.dat");
+
+ }
+
+ @Test
+ public void testNonRecursive() throws Exception {
+
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from(fileUri(
+ "d1?fileName=d1.dat&readLock=markerFile&readLockDeleteOrphanLockFiles=true&initialDelay=0&delay=10"))
+ .to("mock:result");
+ }
+ });
+ context.start();
+
+ NotifyBuilder notify = new NotifyBuilder(context).whenDone(1).create();
+ boolean done = notify.matches(5, TimeUnit.SECONDS);
+ assertTrue(done, "Route should be done processing 1 exchanges");
+ checkFilesNotExists("d1", "d1.dat");
+ checkFilesExists("d1/d2", "d2.dat");
+ checkFilesExists("d1/d2/d3", "d3.dat");
+ checkFilesExists("d1/d2/d3/d4", "d4.dat");
+ checkFilesExists("d1/d2/d3/d4/d5", "d5.dat");
+ }
+
+ @Test
+ public void testRecursiveSingleDepth() throws Exception {
+
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from(fileUri(
+ "d1?include=.*.dat&readLock=markerFile&readLockDeleteOrphanLockFiles=true&initialDelay=0&delay=10&recursive=true&minDepth=2&maxDepth=2"))
+ .to("mock:result");
+ }
+ });
+ context.start();
+
+ NotifyBuilder notify = new NotifyBuilder(context).whenDone(1).create();
+
+ boolean done = notify.matches(5, TimeUnit.SECONDS);
+ assertTrue(done, "Route should be done processing 1 exchanges");
+
+ context.stop();
+
+ checkFilesExists("d1", "d1.dat");
+ checkFilesNotExists("d1/d2", "d2.dat");
+ checkFilesExists("d1/d2/d3", "d3.dat");
+ checkFilesExists("d1/d2/d3/d4", "d4.dat");
+ checkFilesExists("d1/d2/d3/d4/d5", "d5.dat");
+
+ }
+
+ @Test
+ public void testRecursiveRange() throws Exception {
+
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from(fileUri(
+ "d1?include=.*.dat&readLock=markerFile&readLockDeleteOrphanLockFiles=true&initialDelay=0&delay=10&recursive=true&minDepth=2&maxDepth=4"))
+ .to("mock:result");
+ }
+ });
+ context.start();
+
+ NotifyBuilder notify = new NotifyBuilder(context).whenDone(3).create();
+ boolean done = notify.matches(5, TimeUnit.SECONDS);
+ assertTrue(done, "Route should be done processing 3 exchanges");
+
+ context.stop();
+
+ checkFilesExists("d1", "d1.dat");
+ checkFilesNotExists("d1/d2", "d2.dat");
+ checkFilesNotExists("d1/d2/d3", "d3.dat");
+ checkFilesNotExists("d1/d2/d3/d4", "d4.dat");
+ checkFilesExists("d1/d2/d3/d4/d5", "d5.dat");
+
+ }
+
+ @Test
+ public void testRecursiveRangeAntInclude() throws Exception {
+
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from(fileUri(
+ "d1?antInclude=**/*.dat&readLock=markerFile&readLockDeleteOrphanLockFiles=true&initialDelay=0&delay=10&recursive=true&minDepth=2&maxDepth=4"))
+ .to("mock:result");
+ }
+ });
+ context.start();
+
+ NotifyBuilder notify = new NotifyBuilder(context).whenDone(3).create();
+ boolean done = notify.matches(5, TimeUnit.SECONDS);
+ assertTrue(done, "Route should be done processing 3 exchanges");
+
+ context.stop();
+
+ checkFilesExists("d1", "d1.dat");
+ checkFilesNotExists("d1/d2", "d2.dat");
+ checkFilesNotExists("d1/d2/d3", "d3.dat");
+ checkFilesNotExists("d1/d2/d3/d4", "d4.dat");
+ checkFilesExists("d1/d2/d3/d4/d5", "d5.dat");
+
+ }
+
+ @Test
+ public void testRecursive() throws Exception {
+
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from(fileUri(
+ "d1?include=.*.dat&readLock=markerFile&readLockDeleteOrphanLockFiles=true&initialDelay=0&delay=10&recursive=true"))
+ .to("mock:result");
+ }
+ });
+ context.start();
+
+ NotifyBuilder notify = new NotifyBuilder(context).whenDone(5).create();
+ boolean done = notify.matches(5, TimeUnit.SECONDS);
+ assertTrue(done, "Route should be done processing 5 exchanges");
+
+ context.stop();
+
+ checkFilesNotExists("d1", "d1.dat");
+ checkFilesNotExists("d1/d2", "d2.dat");
+ checkFilesNotExists("d1/d2/d3", "d3.dat");
+ checkFilesNotExists("d1/d2/d3/d4", "d4.dat");
+ checkFilesNotExists("d1/d2/d3/d4/d5", "d5.dat");
+
+ }
+
+ private void createFiles(String dir, String fileName) throws IOException {
+ Files.write(testFile(dir + "/" + fileName), fileName.getBytes());
+ Files.write(testFile(dir + "/" + fileName + FileComponent.DEFAULT_LOCK_FILE_POSTFIX), "".getBytes());
+ }
+
+ private void checkFilesExists(String dir, String fileName) {
+ assertFileExists(testFile(dir + "/" + fileName));
+ assertFileExists(testFile(dir + "/" + fileName + FileComponent.DEFAULT_LOCK_FILE_POSTFIX));
+ }
+
+ private void checkFilesNotExists(String dir, String fileName) {
+ assertFileNotExists(testFile(dir + "/" + fileName));
+ assertFileNotExists(testFile(dir + "/" + fileName + FileComponent.DEFAULT_LOCK_FILE_POSTFIX));
+ }
+
+}