You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2022/06/20 13:10:06 UTC
[commons-io] branch master updated (bef96f6e -> a6f188ed)
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/commons-io.git
from bef96f6e Add IOBiFunction.noop()
new cddc925e Add option for AccumulatorPathVisitor to ignore file visitation failures
new a6f188ed [IO-755] Using FileUtils.listFiles() with background changes fails on Linux
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/main/java/org/apache/commons/io/FileUtils.java | 3 +-
.../commons/io/file/AccumulatorPathVisitor.java | 16 +++
.../commons/io/file/CountingPathVisitor.java | 29 +++++-
.../apache/commons/io/file/NoopPathVisitor.java | 24 +++++
.../apache/commons/io/file/SimplePathVisitor.java | 21 ++++
.../io/file/AccumulatorPathVisitorTest.java | 116 +++++++++++++++++++++
6 files changed, 207 insertions(+), 2 deletions(-)
[commons-io] 02/02: [IO-755] Using FileUtils.listFiles() with background changes fails on Linux
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-io.git
commit a6f188edaf20796489f6fede3f4b25ffcf35956c
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Jun 20 09:09:15 2022 -0400
[IO-755] Using FileUtils.listFiles() with background changes fails on
Linux
---
src/main/java/org/apache/commons/io/FileUtils.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/apache/commons/io/FileUtils.java b/src/main/java/org/apache/commons/io/FileUtils.java
index aa747983..5072a277 100644
--- a/src/main/java/org/apache/commons/io/FileUtils.java
+++ b/src/main/java/org/apache/commons/io/FileUtils.java
@@ -78,6 +78,7 @@ import org.apache.commons.io.filefilter.FileFileFilter;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.SuffixFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
import org.apache.commons.io.function.IOConsumer;
/**
@@ -2174,7 +2175,7 @@ public class FileUtils {
final boolean isDirFilterSet = dirFilter != null;
final FileEqualsFileFilter rootDirFilter = new FileEqualsFileFilter(directory);
final PathFilter dirPathFilter = isDirFilterSet ? rootDirFilter.or(dirFilter) : rootDirFilter;
- final AccumulatorPathVisitor visitor = new AccumulatorPathVisitor(Counters.noopPathCounters(), fileFilter, dirPathFilter);
+ final AccumulatorPathVisitor visitor = new AccumulatorPathVisitor(Counters.noopPathCounters(), fileFilter, dirPathFilter, IOBiFunction.noop());
final Set<FileVisitOption> optionSet = new HashSet<>();
Collections.addAll(optionSet, options);
Files.walkFileTree(directory.toPath(), optionSet, toMaxDepth(isDirFilterSet), visitor);
[commons-io] 01/02: Add option for AccumulatorPathVisitor to ignore file visitation failures
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-io.git
commit cddc925ea8ec7b37d86df5a067a65035e0892d6e
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Jun 20 09:08:08 2022 -0400
Add option for AccumulatorPathVisitor to ignore file visitation failures
---
.../commons/io/file/AccumulatorPathVisitor.java | 16 +++
.../commons/io/file/CountingPathVisitor.java | 29 +++++-
.../apache/commons/io/file/NoopPathVisitor.java | 24 +++++
.../apache/commons/io/file/SimplePathVisitor.java | 21 ++++
.../io/file/AccumulatorPathVisitorTest.java | 116 +++++++++++++++++++++
5 files changed, 205 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java b/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
index c5d2b0de..0574ddd7 100644
--- a/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
@@ -18,6 +18,7 @@
package org.apache.commons.io.file;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
@@ -26,6 +27,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.commons.io.file.Counters.PathCounters;
+import org.apache.commons.io.function.IOBiFunction;
/**
* Accumulates normalized paths during visitation.
@@ -135,6 +137,20 @@ public class AccumulatorPathVisitor extends CountingPathVisitor {
super(pathCounter, fileFilter, dirFilter);
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param pathCounter How to count path visits.
+ * @param fileFilter Filters which files to count.
+ * @param dirFilter Filters which directories to count.
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+ final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(pathCounter, fileFilter, dirFilter, visitFileFailed);
+ }
+
private void add(final List<Path> list, final Path dir) {
list.add(dir.normalize());
}
diff --git a/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java b/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
index c0ab0cba..b841019d 100644
--- a/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
@@ -26,8 +26,10 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.Objects;
import org.apache.commons.io.file.Counters.PathCounters;
+import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.SymbolicLinkFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
/**
* Counts files, directories, and sizes, as a visit proceeds.
@@ -38,6 +40,14 @@ public class CountingPathVisitor extends SimplePathVisitor {
static final String[] EMPTY_STRING_ARRAY = {};
+ static IOFileFilter defaultDirFilter() {
+ return TrueFileFilter.INSTANCE;
+ }
+
+ static IOFileFilter defaultFileFilter() {
+ return new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE);
+ }
+
/**
* Creates a new instance configured with a {@link BigInteger} {@link PathCounters}.
*
@@ -66,7 +76,7 @@ public class CountingPathVisitor extends SimplePathVisitor {
* @param pathCounter How to count path visits.
*/
public CountingPathVisitor(final PathCounters pathCounter) {
- this(pathCounter, new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE), TrueFileFilter.INSTANCE);
+ this(pathCounter, defaultFileFilter(), defaultDirFilter());
}
/**
@@ -83,6 +93,23 @@ public class CountingPathVisitor extends SimplePathVisitor {
this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param pathCounter How to count path visits.
+ * @param fileFilter Filters which files to count.
+ * @param dirFilter Filters which directories to count.
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public CountingPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+ final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(visitFileFailed);
+ this.pathCounters = Objects.requireNonNull(pathCounter, "pathCounter");
+ this.fileFilter = Objects.requireNonNull(fileFilter, "fileFilter");
+ this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
+ }
+
@Override
public boolean equals(final Object obj) {
if (this == obj) {
diff --git a/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java b/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
index 48a13239..8fc07d5f 100644
--- a/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
@@ -17,6 +17,12 @@
package org.apache.commons.io.file;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+
+import org.apache.commons.io.function.IOBiFunction;
+
/**
* A noop path visitor.
*
@@ -28,4 +34,22 @@ public class NoopPathVisitor extends SimplePathVisitor {
* The singleton instance.
*/
public static final NoopPathVisitor INSTANCE = new NoopPathVisitor();
+
+ /**
+ * Constructs a new instance.
+ *
+ * @since 2.12.0
+ */
+ public NoopPathVisitor() {
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public NoopPathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(visitFileFailed);
+ }
}
diff --git a/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java b/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
index b4bb2d9e..1c3b1ddc 100644
--- a/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
@@ -17,8 +17,13 @@
package org.apache.commons.io.file;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
+import java.util.Objects;
+
+import org.apache.commons.io.function.IOBiFunction;
/**
* A {@link SimpleFileVisitor} typed to a {@link Path}.
@@ -27,10 +32,26 @@ import java.nio.file.SimpleFileVisitor;
*/
public abstract class SimplePathVisitor extends SimpleFileVisitor<Path> implements PathVisitor {
+ private final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailedFunction;
+
/**
* Constructs a new instance.
*/
protected SimplePathVisitor() {
+ this.visitFileFailedFunction = super::visitFileFailed;
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ */
+ protected SimplePathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ this.visitFileFailedFunction = Objects.requireNonNull(visitFileFailed, "visitFileFailed");
}
+ @Override
+ public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException {
+ return visitFileFailedFunction.apply(file, exc);
+ }
}
diff --git a/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java b/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
index b03aaaae..06903a1e 100644
--- a/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
+++ b/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
@@ -22,9 +22,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Stream;
@@ -32,6 +43,7 @@ import org.apache.commons.io.filefilter.AndFileFilter;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.EmptyFileFilter;
import org.apache.commons.io.filefilter.PathVisitorFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -50,6 +62,17 @@ public class AccumulatorPathVisitorTest {
// @formatter:on
}
+ static Stream<Arguments> testParametersIgnoreFailures() {
+ // @formatter:off
+ return Stream.of(
+ Arguments.of((Supplier<AccumulatorPathVisitor>) () -> new AccumulatorPathVisitor(
+ Counters.bigIntegerPathCounters(),
+ CountingPathVisitor.defaultDirFilter(),
+ CountingPathVisitor.defaultFileFilter(),
+ IOBiFunction.noop())));
+ // @formatter:on
+ }
+
@TempDir
Path tempDirPath;
@@ -109,4 +132,97 @@ public class AccumulatorPathVisitorTest {
assertEquals(2, accPathVisitor.getFileList().size());
}
+ /**
+ * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit.
+ *
+ * Random failure like:
+ *
+ * <pre>
+ * ...?...
+ * </pre>
+ */
+ @ParameterizedTest
+ @MethodSource("testParametersIgnoreFailures")
+ public void testFolderWhileDeletingAsync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException, InterruptedException {
+ final int count = 10_000;
+ final List<Path> files = new ArrayList<>(count);
+ // Create "count" file fixtures
+ for (int i = 1; i <= count; i++) {
+ final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt");
+ assertTrue(Files.exists(tempFile));
+ files.add(tempFile);
+ }
+ final AccumulatorPathVisitor accPathVisitor = supplier.get();
+ final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) {
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException {
+ // Slow down the walking a bit to try and cause conflicts with the deletion thread
+ try {
+ Thread.sleep(10);
+ } catch (final InterruptedException ignore) {
+ // e.printStackTrace();
+ }
+ return super.visitFile(path, attributes);
+ }
+ };
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ final AtomicBoolean deleted = new AtomicBoolean();
+ try {
+ executor.execute(() -> {
+ for (final Path file : files) {
+ try {
+ // File deletion is slow compared to tree walking, so we go as fast as we can here
+ Files.delete(file);
+ } catch (final IOException ignored) {
+ // e.printStackTrace();
+ }
+ }
+ deleted.set(true);
+ });
+ Files.walkFileTree(tempDirPath, countingFileFilter);
+ } finally {
+ if (!deleted.get()) {
+ Thread.sleep(1000);
+ }
+ if (!deleted.get()) {
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ executor.shutdownNow();
+ }
+ }
+
+ /**
+ * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit.
+ */
+ @ParameterizedTest
+ @MethodSource("testParametersIgnoreFailures")
+ public void testFolderWhileDeletingSync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException {
+ final int count = 100;
+ final int marker = count / 2;
+ final Set<Path> files = new LinkedHashSet<>(count);
+ for (int i = 1; i <= count; i++) {
+ final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt");
+ assertTrue(Files.exists(tempFile));
+ files.add(tempFile);
+ }
+ final AccumulatorPathVisitor accPathVisitor = supplier.get();
+ final AtomicInteger visitCount = new AtomicInteger();
+ final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) {
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException {
+ if (visitCount.incrementAndGet() == marker) {
+ // Now that we've visited half the files, delete them all
+ for (final Path file : files) {
+ Files.delete(file);
+ }
+ }
+ return super.visitFile(path, attributes);
+ }
+ };
+ Files.walkFileTree(tempDirPath, countingFileFilter);
+ assertCounts(1, marker - 1, 0, accPathVisitor.getPathCounters());
+ assertEquals(1, accPathVisitor.getDirList().size());
+ assertEquals(marker - 1, accPathVisitor.getFileList().size());
+ }
+
}