You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/11/03 09:58:46 UTC

Change in asterixdb[master]: [NO ISSUE][OTH] Replace usage of FileUtils.listFiles

From Murtadha Hubail <mh...@apache.org>:

Murtadha Hubail has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13923 )


Change subject: [NO ISSUE][OTH] Replace usage of FileUtils.listFiles
......................................................................

[NO ISSUE][OTH] Replace usage of FileUtils.listFiles

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- Avoid depending on FileUtils.listFiles in production code
  by replacing it with our own implementation.

Change-Id: I1ecb6143d5840fc12b35a641e1bc73c40bba3676
---
M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
2 files changed, 44 insertions(+), 40 deletions(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/23/13923/1

diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 02f5772..6577367 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -60,7 +60,6 @@
 import org.apache.asterix.common.utils.StorageConstants;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.api.io.IIOManager;
@@ -96,41 +95,10 @@
     private static final FilenameFilter MASK_FILES_FILTER =
             (dir, name) -> name.startsWith(StorageConstants.MASK_FILE_PREFIX);
     private static final int MAX_CACHED_RESOURCES = 1000;
-    private static final IOFileFilter METADATA_FILES_FILTER = new IOFileFilter() {
-        @Override
-        public boolean accept(File file) {
-            return file.getName().equals(StorageConstants.METADATA_FILE_NAME);
-        }
-
-        @Override
-        public boolean accept(File dir, String name) {
-            return false;
-        }
-    };
-
-    private static final IOFileFilter METADATA_MASK_FILES_FILTER = new IOFileFilter() {
-        @Override
-        public boolean accept(File file) {
-            return file.getName().equals(METADATA_FILE_MASK_NAME);
-        }
-
-        @Override
-        public boolean accept(File dir, String name) {
-            return false;
-        }
-    };
-
-    private static final IOFileFilter ALL_DIR_FILTER = new IOFileFilter() {
-        @Override
-        public boolean accept(File file) {
-            return true;
-        }
-
-        @Override
-        public boolean accept(File dir, String name) {
-            return true;
-        }
-    };
+    private static final FilenameFilter METADATA_FILES_FILTER =
+            (dir, name) -> name.equals(StorageConstants.METADATA_FILE_NAME);
+    private static final FilenameFilter METADATA_MASK_FILES_FILTER =
+            (dir, name) -> name.equals(METADATA_FILE_MASK_NAME);
 
     // Finals
     private final IIOManager ioManager;
@@ -270,7 +238,7 @@
             if (!Files.exists(root) || !Files.isDirectory(root)) {
                 continue;
             }
-            final Collection<File> files = FileUtils.listFiles(root.toFile(), METADATA_FILES_FILTER, ALL_DIR_FILTER);
+            final Collection<File> files = IoUtil.getMatchingFiles(root, METADATA_FILES_FILTER);
             try {
                 for (File file : files) {
                     final LocalResource localResource = readLocalResource(file);
@@ -301,7 +269,7 @@
 
     public synchronized void deleteInvalidIndexes(Predicate<LocalResource> filter) throws HyracksDataException {
         for (Path root : storageRoots) {
-            final Collection<File> files = FileUtils.listFiles(root.toFile(), METADATA_FILES_FILTER, ALL_DIR_FILTER);
+            final Collection<File> files = IoUtil.getMatchingFiles(root, METADATA_FILES_FILTER);
             try {
                 for (File file : files) {
                     final LocalResource localResource = readLocalResource(file);
@@ -519,8 +487,7 @@
 
     public synchronized void deleteCorruptedResources() throws HyracksDataException {
         for (Path root : storageRoots) {
-            final Collection<File> metadataMaskFiles =
-                    FileUtils.listFiles(root.toFile(), METADATA_MASK_FILES_FILTER, ALL_DIR_FILTER);
+            final Collection<File> metadataMaskFiles = IoUtil.getMatchingFiles(root, METADATA_MASK_FILES_FILTER);
             for (File metadataMaskFile : metadataMaskFiles) {
                 final File resourceFile = new File(metadataMaskFile.getParent(), METADATA_FILE_NAME);
                 if (resourceFile.exists()) {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
index 6ad53ab..7e787b6 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/IoUtil.java
@@ -19,13 +19,18 @@
 package org.apache.hyracks.api.util;
 
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Objects;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -152,4 +157,36 @@
             }
         }
     }
+
+    /**
+     * Gets a collection of files matching {@code filter} by searching {@code root} directory and
+     * all of its subdirectories
+     *
+     * @param root
+     * @param filter
+     * @return a collection of matching files
+     */
+    public static Collection<File> getMatchingFiles(Path root, FilenameFilter filter) {
+        if (!Files.isDirectory(root)) {
+            throw new IllegalArgumentException("Parameter 'directory' is not a directory: " + root);
+        }
+        Objects.requireNonNull(filter);
+        Collection<File> files = new ArrayList<>();
+        FileFilter dirOrMatchingFileFilter = file -> file.isDirectory() || filter.accept(file, file.getName());
+        collectDirFiles(root.toFile(), dirOrMatchingFileFilter, files);
+        return files;
+    }
+
+    private static void collectDirFiles(File dir, FileFilter filter, Collection<File> files) {
+        File[] listFiles = dir.listFiles(filter);
+        if (listFiles != null) {
+            for (File file : listFiles) {
+                if (file.isDirectory()) {
+                    collectDirFiles(file, filter, files);
+                } else {
+                    files.add(file);
+                }
+            }
+        }
+    }
 }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13923
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I1ecb6143d5840fc12b35a641e1bc73c40bba3676
Gerrit-Change-Number: 13923
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-MessageType: newchange