You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2022/06/20 13:10:07 UTC
[commons-io] 01/02: Add option for AccumulatorPathVisitor to ignore file visitation failures
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-io.git
commit cddc925ea8ec7b37d86df5a067a65035e0892d6e
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Jun 20 09:08:08 2022 -0400
Add option for AccumulatorPathVisitor to ignore file visitation failures
---
.../commons/io/file/AccumulatorPathVisitor.java | 16 +++
.../commons/io/file/CountingPathVisitor.java | 29 +++++-
.../apache/commons/io/file/NoopPathVisitor.java | 24 +++++
.../apache/commons/io/file/SimplePathVisitor.java | 21 ++++
.../io/file/AccumulatorPathVisitorTest.java | 116 +++++++++++++++++++++
5 files changed, 205 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java b/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
index c5d2b0de..0574ddd7 100644
--- a/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
@@ -18,6 +18,7 @@
package org.apache.commons.io.file;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
@@ -26,6 +27,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.commons.io.file.Counters.PathCounters;
+import org.apache.commons.io.function.IOBiFunction;
/**
* Accumulates normalized paths during visitation.
@@ -135,6 +137,20 @@ public class AccumulatorPathVisitor extends CountingPathVisitor {
super(pathCounter, fileFilter, dirFilter);
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param pathCounter How to count path visits.
+ * @param fileFilter Filters which files to count.
+ * @param dirFilter Filters which directories to count.
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+ final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(pathCounter, fileFilter, dirFilter, visitFileFailed);
+ }
+
private void add(final List<Path> list, final Path dir) {
list.add(dir.normalize());
}
diff --git a/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java b/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
index c0ab0cba..b841019d 100644
--- a/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
@@ -26,8 +26,10 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.Objects;
import org.apache.commons.io.file.Counters.PathCounters;
+import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.SymbolicLinkFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
/**
* Counts files, directories, and sizes, as a visit proceeds.
@@ -38,6 +40,14 @@ public class CountingPathVisitor extends SimplePathVisitor {
static final String[] EMPTY_STRING_ARRAY = {};
+ static IOFileFilter defaultDirFilter() {
+ return TrueFileFilter.INSTANCE;
+ }
+
+ static IOFileFilter defaultFileFilter() {
+ return new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE);
+ }
+
/**
* Creates a new instance configured with a {@link BigInteger} {@link PathCounters}.
*
@@ -66,7 +76,7 @@ public class CountingPathVisitor extends SimplePathVisitor {
* @param pathCounter How to count path visits.
*/
public CountingPathVisitor(final PathCounters pathCounter) {
- this(pathCounter, new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE), TrueFileFilter.INSTANCE);
+ this(pathCounter, defaultFileFilter(), defaultDirFilter());
}
/**
@@ -83,6 +93,23 @@ public class CountingPathVisitor extends SimplePathVisitor {
this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param pathCounter How to count path visits.
+ * @param fileFilter Filters which files to count.
+ * @param dirFilter Filters which directories to count.
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public CountingPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+ final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(visitFileFailed);
+ this.pathCounters = Objects.requireNonNull(pathCounter, "pathCounter");
+ this.fileFilter = Objects.requireNonNull(fileFilter, "fileFilter");
+ this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
+ }
+
@Override
public boolean equals(final Object obj) {
if (this == obj) {
diff --git a/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java b/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
index 48a13239..8fc07d5f 100644
--- a/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
@@ -17,6 +17,12 @@
package org.apache.commons.io.file;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+
+import org.apache.commons.io.function.IOBiFunction;
+
/**
* A noop path visitor.
*
@@ -28,4 +34,22 @@ public class NoopPathVisitor extends SimplePathVisitor {
* The singleton instance.
*/
public static final NoopPathVisitor INSTANCE = new NoopPathVisitor();
+
+ /**
+ * Constructs a new instance.
+ *
+ * @since 2.12.0
+ */
+ public NoopPathVisitor() {
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public NoopPathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(visitFileFailed);
+ }
}
diff --git a/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java b/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
index b4bb2d9e..1c3b1ddc 100644
--- a/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
+++ b/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
@@ -17,8 +17,13 @@
package org.apache.commons.io.file;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
+import java.util.Objects;
+
+import org.apache.commons.io.function.IOBiFunction;
/**
* A {@link SimpleFileVisitor} typed to a {@link Path}.
@@ -27,10 +32,26 @@ import java.nio.file.SimpleFileVisitor;
*/
public abstract class SimplePathVisitor extends SimpleFileVisitor<Path> implements PathVisitor {
+ private final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailedFunction;
+
/**
* Constructs a new instance.
*/
protected SimplePathVisitor() {
+ this.visitFileFailedFunction = super::visitFileFailed;
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ */
+ protected SimplePathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ this.visitFileFailedFunction = Objects.requireNonNull(visitFileFailed, "visitFileFailed");
}
+ @Override
+ public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException {
+ return visitFileFailedFunction.apply(file, exc);
+ }
}
diff --git a/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java b/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
index b03aaaae..06903a1e 100644
--- a/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
+++ b/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
@@ -22,9 +22,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Stream;
@@ -32,6 +43,7 @@ import org.apache.commons.io.filefilter.AndFileFilter;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.EmptyFileFilter;
import org.apache.commons.io.filefilter.PathVisitorFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -50,6 +62,17 @@ public class AccumulatorPathVisitorTest {
// @formatter:on
}
+ static Stream<Arguments> testParametersIgnoreFailures() {
+ // @formatter:off
+ return Stream.of(
+ Arguments.of((Supplier<AccumulatorPathVisitor>) () -> new AccumulatorPathVisitor(
+ Counters.bigIntegerPathCounters(),
+ CountingPathVisitor.defaultDirFilter(),
+ CountingPathVisitor.defaultFileFilter(),
+ IOBiFunction.noop())));
+ // @formatter:on
+ }
+
@TempDir
Path tempDirPath;
@@ -109,4 +132,97 @@ public class AccumulatorPathVisitorTest {
assertEquals(2, accPathVisitor.getFileList().size());
}
+ /**
+ * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit.
+ *
+ * Random failure like:
+ *
+ * <pre>
+ * ...?...
+ * </pre>
+ */
+ @ParameterizedTest
+ @MethodSource("testParametersIgnoreFailures")
+ public void testFolderWhileDeletingAsync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException, InterruptedException {
+ final int count = 10_000;
+ final List<Path> files = new ArrayList<>(count);
+ // Create "count" file fixtures
+ for (int i = 1; i <= count; i++) {
+ final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt");
+ assertTrue(Files.exists(tempFile));
+ files.add(tempFile);
+ }
+ final AccumulatorPathVisitor accPathVisitor = supplier.get();
+ final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) {
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException {
+ // Slow down the walking a bit to try and cause conflicts with the deletion thread
+ try {
+ Thread.sleep(10);
+ } catch (final InterruptedException ignore) {
+ // e.printStackTrace();
+ }
+ return super.visitFile(path, attributes);
+ }
+ };
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ final AtomicBoolean deleted = new AtomicBoolean();
+ try {
+ executor.execute(() -> {
+ for (final Path file : files) {
+ try {
+ // File deletion is slow compared to tree walking, so we go as fast as we can here
+ Files.delete(file);
+ } catch (final IOException ignored) {
+ // e.printStackTrace();
+ }
+ }
+ deleted.set(true);
+ });
+ Files.walkFileTree(tempDirPath, countingFileFilter);
+ } finally {
+ if (!deleted.get()) {
+ Thread.sleep(1000);
+ }
+ if (!deleted.get()) {
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ executor.shutdownNow();
+ }
+ }
+
+ /**
+ * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit.
+ */
+ @ParameterizedTest
+ @MethodSource("testParametersIgnoreFailures")
+ public void testFolderWhileDeletingSync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException {
+ final int count = 100;
+ final int marker = count / 2;
+ final Set<Path> files = new LinkedHashSet<>(count);
+ for (int i = 1; i <= count; i++) {
+ final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt");
+ assertTrue(Files.exists(tempFile));
+ files.add(tempFile);
+ }
+ final AccumulatorPathVisitor accPathVisitor = supplier.get();
+ final AtomicInteger visitCount = new AtomicInteger();
+ final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) {
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException {
+ if (visitCount.incrementAndGet() == marker) {
+ // Now that we've visited half the files, delete them all
+ for (final Path file : files) {
+ Files.delete(file);
+ }
+ }
+ return super.visitFile(path, attributes);
+ }
+ };
+ Files.walkFileTree(tempDirPath, countingFileFilter);
+ assertCounts(1, marker - 1, 0, accPathVisitor.getPathCounters());
+ assertEquals(1, accPathVisitor.getDirList().size());
+ assertEquals(marker - 1, accPathVisitor.getFileList().size());
+ }
+
}