You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by GitBox <gi...@apache.org> on 2018/11/26 16:04:23 UTC
[GitHub] asfgit closed pull request #1547: DRILL-6858: Add functionality to
list directories / files with exceptions suppression
asfgit closed pull request #1547: DRILL-6858: Add functionality to list directories / files with exceptions suppression
URL: https://github.com/apache/drill/pull/1547
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
index 75abfdd65be..9782bbff6b8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
@@ -32,7 +32,6 @@
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.hadoop.fs.Path;
-import java.io.IOException;
import java.sql.Timestamp;
import java.util.List;
import java.util.stream.Collectors;
@@ -46,7 +45,7 @@ public ShowFilesHandler(SqlHandlerConfig config) {
}
@Override
- public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException, IOException {
+ public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException {
SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
SchemaPlus drillSchema = defaultSchema;
SqlShowFiles showFiles = unwrap(sqlNode, SqlShowFiles.class);
@@ -83,7 +82,7 @@ public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException, IOExc
}
Path path = new Path(wsSchema.getDefaultLocation(), fromDir);
- List<ShowFilesCommandResult> records = FileSystemUtil.listAll(wsSchema.getFS(), path, false).stream()
+ List<ShowFilesCommandResult> records = FileSystemUtil.listAllSafe(wsSchema.getFS(), path, false).stream()
// use ShowFilesCommandResult for backward compatibility
.map(fileStatus -> new ShowFilesCommandResult(new Records.File(wsSchema.getFullSchemaName(), wsSchema, fileStatus)))
.collect(Collectors.toList());
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaRecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaRecordGenerator.java
index cfb918b91e9..1e728403d31 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaRecordGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaRecordGenerator.java
@@ -31,7 +31,6 @@
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.SHRD_COL_TABLE_SCHEMA;
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.TBLS_COL_TABLE_TYPE;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -436,14 +435,12 @@ public void visitFiles(String schemaName, SchemaPlus schemaPlus) {
String defaultLocation = wsSchema.getDefaultLocation();
FileSystem fs = wsSchema.getFS();
boolean recursive = optionManager.getBoolean(ExecConstants.LIST_FILES_RECURSIVELY);
- FileSystemUtil.listAll(fs, new Path(defaultLocation), recursive).forEach(
+ FileSystemUtil.listAllSafe(fs, new Path(defaultLocation), recursive).forEach(
fileStatus -> records.add(new Records.File(schemaName, wsSchema, fileStatus))
);
}
} catch (ClassCastException | UnsupportedOperationException e) {
// ignore the exception since either this is not a Drill schema or schema does not support files listing
- } catch (IOException e) {
- logger.warn("Failure while trying to list files", e);
}
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DrillFileSystemUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DrillFileSystemUtil.java
index 2fc135d4d37..fcee3b0e075 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DrillFileSystemUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DrillFileSystemUtil.java
@@ -36,12 +36,8 @@
/**
* Path filter that skips all files and folders that start with dot or underscore.
*/
- public static final PathFilter DRILL_SYSTEM_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return !path.getName().startsWith(DrillFileSystem.UNDERSCORE_PREFIX) && !path.getName().startsWith(DrillFileSystem.DOT_PREFIX);
- }
- };
+ public static final PathFilter DRILL_SYSTEM_FILTER = path ->
+ !path.getName().startsWith(DrillFileSystem.UNDERSCORE_PREFIX) && !path.getName().startsWith(DrillFileSystem.DOT_PREFIX);
/**
* Returns statuses of all directories present in given path applying custom filters if present.
@@ -58,6 +54,22 @@ public boolean accept(Path path) {
return FileSystemUtil.listDirectories(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}
+ /**
+ * Returns statuses of all directories present in given path applying custom filters if present.
+ * Directories that start with dot or underscore are skipped.
+ * Will include nested directories if recursive flag is set to true.
+ * Will ignore all exceptions during listing if any.
+ *
+ * @param fs current file system
+ * @param path path to directory
+ * @param recursive true if nested directories should be included
+ * @param filters list of custom filters (optional)
+ * @return list of matching directory statuses
+ */
+ public static List<FileStatus> listDirectoriesSafe(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
+ return FileSystemUtil.listDirectoriesSafe(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
+ }
+
/**
* Returns statuses of all files present in given path applying custom filters if present.
* Files and nested directories that start with dot or underscore are skipped.
@@ -73,6 +85,23 @@ public boolean accept(Path path) {
return FileSystemUtil.listFiles(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}
+ /**
+ * Returns statuses of all files present in given path applying custom filters if present.
+ * Files and nested directories that start with dot or underscore are skipped.
+ * Will include files from nested directories if recursive flag is set to true.
+ * Will ignore all exceptions during listing if any.
+ *
+ * @param fs current file system
+ * @param path path to file or directory
+ * @param recursive true if files in nested directories should be included
+ * @param filters list of custom filters (optional)
+ * @return list of matching file statuses
+ */
+ public static List<FileStatus> listFilesSafe(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
+ return FileSystemUtil.listFilesSafe(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
+ }
+
+
/**
* Returns statuses of all directories and files present in given path applying custom filters if present.
* Directories and files that start with dot or underscore are skipped.
@@ -88,4 +117,20 @@ public boolean accept(Path path) {
return FileSystemUtil.listAll(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}
+ /**
+ * Returns statuses of all directories and files present in given path applying custom filters if present.
+ * Directories and files that start with dot or underscore are skipped.
+ * Will include nested directories and their files if recursive flag is set to true.
+ * Will ignore all exceptions during listing if any.
+ *
+ * @param fs current file system
+ * @param path path to file or directory
+ * @param recursive true if nested directories and their files should be included
+ * @param filters list of custom filters (optional)
+ * @return list of matching directory and file statuses
+ */
+ public static List<FileStatus> listAllSafe(FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
+ return FileSystemUtil.listAllSafe(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
+ }
+
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
index f17e215e1a9..47ac44c0755 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
@@ -26,6 +26,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.stream.Stream;
/**
* Helper class that provides methods to list directories or file or both statuses.
@@ -33,15 +34,12 @@
*/
public class FileSystemUtil {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemUtil.class);
+
/**
* Filter that will accept all files and directories.
*/
- public static final PathFilter DUMMY_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return true;
- }
- };
+ public static final PathFilter DUMMY_FILTER = path -> true;
/**
* Returns statuses of all directories present in given path applying custom filters if present.
@@ -55,7 +53,28 @@ public boolean accept(Path path) {
*/
public static List<FileStatus> listDirectories(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) throws IOException {
List<FileStatus> statuses = new ArrayList<>();
- listDirectories(fs, path, recursive, statuses, mergeFilters(filters));
+ listDirectories(fs, path, recursive, false, statuses, mergeFilters(filters));
+ return statuses;
+ }
+
+ /**
+ * Returns statuses of all directories present in given path applying custom filters if present.
+ * Will also include nested directories if recursive flag is set to true.
+ * Will ignore all exceptions during listing if any.
+ *
+ * @param fs current file system
+ * @param path path to directory
+ * @param recursive true if nested directories should be included
+ * @param filters list of custom filters (optional)
+ * @return list of matching directory statuses
+ */
+ public static List<FileStatus> listDirectoriesSafe(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
+ List<FileStatus> statuses = new ArrayList<>();
+ try {
+ listDirectories(fs, path, recursive, true, statuses, mergeFilters(filters));
+ } catch (Exception e) {
+ // all exceptions are ignored
+ }
return statuses;
}
@@ -71,7 +90,27 @@ public boolean accept(Path path) {
*/
public static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recursive, PathFilter... filters) throws IOException {
List<FileStatus> statuses = new ArrayList<>();
- listFiles(fs, path, recursive, statuses, mergeFilters(filters));
+ listFiles(fs, path, recursive, false, statuses, mergeFilters(filters));
+ return statuses;
+ }
+
+ /**
+ * Returns statuses of all files present in given path applying custom filters if present.
+ * Will also include files from nested directories if recursive flag is set to true.
+ *
+ * @param fs current file system
+ * @param path path to file or directory
+ * @param recursive true if files in nested directories should be included
+ * @param filters list of custom filters (optional)
+ * @return list of matching file statuses
+ */
+ public static List<FileStatus> listFilesSafe(FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
+ List<FileStatus> statuses = new ArrayList<>();
+ try {
+ listFiles(fs, path, recursive, true, statuses, mergeFilters(filters));
+ } catch (Exception e) {
+ // all exceptions are ignored
+ }
return statuses;
}
@@ -87,7 +126,28 @@ public boolean accept(Path path) {
*/
public static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursive, PathFilter... filters) throws IOException {
List<FileStatus> statuses = new ArrayList<>();
- listAll(fs, path, recursive, statuses, mergeFilters(filters));
+ listAll(fs, path, recursive, false, statuses, mergeFilters(filters));
+ return statuses;
+ }
+
+ /**
+ * Returns statuses of all directories and files present in given path applying custom filters if present.
+ * Will also include nested directories and their files if recursive flag is set to true.
+ * Will ignore all exceptions during listing if any.
+ *
+ * @param fs current file system
+ * @param path path to file or directory
+ * @param recursive true if nested directories and their files should be included
+ * @param filters list of custom filters (optional)
+ * @return list of matching directory and file statuses
+ */
+ public static List<FileStatus> listAllSafe(FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
+ List<FileStatus> statuses = new ArrayList<>();
+ try {
+ listAll(fs, path, recursive, true, statuses, mergeFilters(filters));
+ } catch (Exception e) {
+ // all exceptions are ignored
+ }
return statuses;
}
@@ -122,39 +182,39 @@ public static PathFilter mergeFilters(final PathFilter... filters) {
return DUMMY_FILTER;
}
- return new PathFilter() {
- @Override
- public boolean accept(Path path) {
- for (PathFilter filter : filters) {
- if (!filter.accept(path)) {
- return false;
- }
- }
- return true;
- }
- };
+ return path -> Stream.of(filters).allMatch(filter -> filter.accept(path));
}
/**
* Helper method that will store in given holder statuses of all directories present in given path applying custom filter.
* If recursive flag is set to true, will call itself recursively to add statuses of nested directories.
+ * If suppress exceptions flag is set to true, will ignore all exceptions during listing.
*
* @param fs current file system
* @param path path to directory
* @param recursive true if nested directories should be included
+ * @param suppressExceptions indicates if exceptions should be ignored during listing
* @param statuses holder for directory statuses
* @param filter custom filter
* @return holder with all matching directory statuses
*/
- private static List<FileStatus> listDirectories(FileSystem fs, Path path, boolean recursive, List<FileStatus> statuses, PathFilter filter) throws IOException {
- FileStatus[] fileStatuses = fs.listStatus(path, filter);
- for (FileStatus status: fileStatuses) {
- if (status.isDirectory()) {
- statuses.add(status);
- if (recursive) {
- listDirectories(fs, status.getPath(), true, statuses, filter);
+ private static List<FileStatus> listDirectories(FileSystem fs, Path path, boolean recursive, boolean suppressExceptions,
+ List<FileStatus> statuses, PathFilter filter) throws IOException {
+ try {
+ for (FileStatus status : fs.listStatus(path, filter)) {
+ if (status.isDirectory()) {
+ statuses.add(status);
+ if (recursive) {
+ listDirectories(fs, status.getPath(), true, suppressExceptions, statuses, filter);
+ }
}
}
+ } catch (Exception e) {
+ if (suppressExceptions) {
+ logger.debug("Exception during listing file statuses", e);
+ } else {
+ throw e;
+ }
}
return statuses;
}
@@ -162,23 +222,33 @@ public boolean accept(Path path) {
/**
* Helper method that will store in given holder statuses of all files present in given path applying custom filter.
* If recursive flag is set to true, will call itself recursively to add file statuses from nested directories.
+ * If suppress exceptions flag is set to true, will ignore all exceptions during listing.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if files in nested directories should be included
+ * @param suppressExceptions indicates if exceptions should be ignored during listing
* @param statuses holder for file statuses
* @param filter custom filter
* @return holder with all matching file statuses
*/
- private static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recursive, List<FileStatus> statuses, PathFilter filter) throws IOException {
- FileStatus[] fileStatuses = fs.listStatus(path, filter);
- for (FileStatus status: fileStatuses) {
- if (status.isDirectory()) {
- if (recursive) {
- listFiles(fs, status.getPath(), true, statuses, filter);
+ private static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recursive, boolean suppressExceptions,
+ List<FileStatus> statuses, PathFilter filter) throws IOException {
+ try {
+ for (FileStatus status : fs.listStatus(path, filter)) {
+ if (status.isDirectory()) {
+ if (recursive) {
+ listFiles(fs, status.getPath(), true, suppressExceptions, statuses, filter);
+ }
+ } else {
+ statuses.add(status);
}
+ }
+ } catch (Exception e) {
+ if (suppressExceptions) {
+ logger.debug("Exception during listing file statuses", e);
} else {
- statuses.add(status);
+ throw e;
}
}
return statuses;
@@ -187,19 +257,30 @@ public boolean accept(Path path) {
/**
* Helper method that will store in given holder statuses of all directories and files present in given path applying custom filter.
* If recursive flag is set to true, will call itself recursively to add nested directories and their file statuses.
+ * If suppress exceptions flag is set to true, will ignore all exceptions during listing.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if nested directories and their files should be included
+ * @param suppressExceptions indicates if exceptions should be ignored during listing
* @param statuses holder for directory and file statuses
* @param filter custom filter
* @return holder with all matching directory and file statuses
*/
- private static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursive, List<FileStatus> statuses, PathFilter filter) throws IOException {
- for (FileStatus status: fs.listStatus(path, filter)) {
- statuses.add(status);
- if (status.isDirectory() && recursive) {
- listAll(fs, status.getPath(), true, statuses, filter);
+ private static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursive, boolean suppressExceptions,
+ List<FileStatus> statuses, PathFilter filter) throws IOException {
+ try {
+ for (FileStatus status : fs.listStatus(path, filter)) {
+ statuses.add(status);
+ if (status.isDirectory() && recursive) {
+ listAll(fs, status.getPath(), true, suppressExceptions, statuses, filter);
+ }
+ }
+ } catch (Exception e) {
+ if (suppressExceptions) {
+ logger.debug("Exception during listing file statuses", e);
+ } else {
+ throw e;
}
}
return statuses;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
index 9acd0154505..507afadda5b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
@@ -30,7 +30,6 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -40,6 +39,7 @@
import java.util.Map;
import static org.hamcrest.core.StringContains.containsString;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -131,10 +131,10 @@ public void testDropTable() throws Exception {
try {
test("drop table parquet_table_700");
} catch (UserException e) {
- Assert.assertTrue(e.getMessage().contains("PERMISSION ERROR"));
+ assertTrue(e.getMessage().contains("PERMISSION ERROR"));
dropFailed = true;
}
- Assert.assertTrue("Permission checking failed during drop table", dropFailed);
+ assertTrue("Permission checking failed during drop table", dropFailed);
}
@Test // DRILL-3037
@@ -176,12 +176,9 @@ public void testShowFilesInWSWithOtherPermissionsForQueryUser() throws Exception
@Test
public void testShowFilesInWSWithNoPermissionsForQueryUser() throws Exception {
updateClient(user2);
-
- thrown.expect(UserRemoteException.class);
- thrown.expectMessage(containsString("Permission denied: user=drillTestUser2, " +
- "access=READ_EXECUTE, inode=\"/drill_test_grp_1_700\":drillTestUser1:drill_test_grp_1:drwx------"));
// Try show tables in schema "drill_test_grp_1_700" which is owned by "user1"
- test("SHOW FILES IN %s.drill_test_grp_1_700", MINI_DFS_STORAGE_PLUGIN_NAME);
+ int count = testSql(String.format("SHOW FILES IN %s.drill_test_grp_1_700", MINI_DFS_STORAGE_PLUGIN_NAME));
+ assertEquals(0, count);
}
@Test
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/DrillFileSystemUtilTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/DrillFileSystemUtilTest.java
index 175929e9605..8ec47be6dd7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/DrillFileSystemUtilTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/DrillFileSystemUtilTest.java
@@ -27,6 +27,7 @@
import java.util.List;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class DrillFileSystemUtilTest extends FileSystemUtilTestBase {
@@ -38,12 +39,8 @@ public void testListDirectoriesWithoutFilter() throws IOException {
@Test
public void testListDirectoriesWithFilter() throws IOException {
- List<FileStatus> statuses = DrillFileSystemUtil.listDirectories(fs, base, false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a");
- }
- });
+ List<FileStatus> statuses = DrillFileSystemUtil.listDirectories(fs, base, false,
+ (PathFilter) path -> path.getName().endsWith("a"));
assertEquals("Directory count should match", 1, statuses.size());
assertEquals("Directory name should match", "a", statuses.get(0).getPath().getName());
}
@@ -56,12 +53,8 @@ public void testListDirectoriesRecursiveWithoutFilter() throws IOException {
@Test
public void testListDirectoriesRecursiveWithFilter() throws IOException {
- List<FileStatus> statuses = DrillFileSystemUtil.listDirectories(fs, base, true, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a");
- }
- });
+ List<FileStatus> statuses = DrillFileSystemUtil.listDirectories(fs, base, true,
+ (PathFilter) path -> path.getName().endsWith("a"));
assertEquals("Directory count should match", 2, statuses.size());
Collections.sort(statuses);
@@ -78,12 +71,8 @@ public void testListFilesWithoutFilter() throws IOException {
@Test
public void testListFilesWithFilter() throws IOException {
- List<FileStatus> statuses = DrillFileSystemUtil.listFiles(fs, new Path(base, "a"), false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = DrillFileSystemUtil.listFiles(fs, new Path(base, "a"), false,
+ (PathFilter) path -> path.getName().endsWith(".txt"));
assertEquals("File count should match", 1, statuses.size());
assertEquals("File name should match", "f.txt", statuses.get(0).getPath().getName());
}
@@ -96,12 +85,8 @@ public void testListFilesRecursiveWithoutFilter() throws IOException {
@Test
public void testListFilesRecursiveWithFilter() throws IOException {
- List<FileStatus> statuses = DrillFileSystemUtil.listFiles(fs, base, true, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a") || path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = DrillFileSystemUtil.listFiles(fs, base, true,
+ (PathFilter) path -> path.getName().endsWith("a") || path.getName().endsWith(".txt"));
assertEquals("File count should match", 2, statuses.size());
Collections.sort(statuses);
@@ -121,12 +106,8 @@ public void testListAllWithoutFilter() throws IOException {
@Test
public void testListAllWithFilter() throws IOException {
- List<FileStatus> statuses = DrillFileSystemUtil.listAll(fs, new Path(base, "a"), false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a") || path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = DrillFileSystemUtil.listAll(fs, new Path(base, "a"), false,
+ (PathFilter) path -> path.getName().endsWith("a") || path.getName().endsWith(".txt"));
assertEquals("Directory and file count should match", 2, statuses.size());
Collections.sort(statuses);
@@ -142,12 +123,8 @@ public void testListAllRecursiveWithoutFilter() throws IOException {
@Test
public void testListAllRecursiveWithFilter() throws IOException {
- List<FileStatus> statuses = DrillFileSystemUtil.listAll(fs, new Path(base, "a"), true, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("a") || path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = DrillFileSystemUtil.listAll(fs, new Path(base, "a"), true,
+ (PathFilter) path -> path.getName().startsWith("a") || path.getName().endsWith(".txt"));
assertEquals("Directory and file count should match", 3, statuses.size());
Collections.sort(statuses);
@@ -156,4 +133,25 @@ public boolean accept(Path path) {
assertEquals("File name should match", "f.txt", statuses.get(2).getPath().getName());
}
+ @Test
+ public void testListDirectoriesSafe() {
+ Path file = new Path(base, "missing");
+ List<FileStatus> fileStatuses = DrillFileSystemUtil.listDirectoriesSafe(fs, file, true);
+ assertTrue("Should return empty result", fileStatuses.isEmpty());
+ }
+
+ @Test
+ public void testListFilesSafe() {
+ Path file = new Path(base, "missing.txt");
+ List<FileStatus> fileStatuses = DrillFileSystemUtil.listFilesSafe(fs, file, true);
+ assertTrue("Should return empty result", fileStatuses.isEmpty());
+ }
+
+ @Test
+ public void testListAllSafe() {
+ Path file = new Path(base, "missing");
+ List<FileStatus> fileStatuses = DrillFileSystemUtil.listAllSafe(fs, file, true);
+ assertTrue("Should return empty result", fileStatuses.isEmpty());
+ }
+
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java
index 764f1d7af4f..299f1cc65e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTest.java
@@ -40,12 +40,8 @@ public void testListDirectoriesWithoutFilter() throws IOException {
@Test
public void testListDirectoriesWithFilter() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listDirectories(fs, base, false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listDirectories(fs, base, false,
+ (PathFilter) path -> path.getName().endsWith("a"));
assertEquals("Directory count should match", 3, statuses.size());
Collections.sort(statuses);
@@ -62,12 +58,8 @@ public void testListDirectoriesRecursiveWithoutFilter() throws IOException {
@Test
public void testListDirectoriesRecursiveWithFilter() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listDirectories(fs, base, true, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listDirectories(fs, base, true,
+ (PathFilter) path -> path.getName().endsWith("a"));
assertEquals("Directory count should match", 4, statuses.size());
Collections.sort(statuses);
@@ -79,12 +71,8 @@ public boolean accept(Path path) {
@Test
public void testListDirectoriesEmptyResult() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listDirectories(fs, base, false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("abc");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listDirectories(fs, base, false,
+ (PathFilter) path -> path.getName().startsWith("abc"));
assertEquals("Directory count should match", 0, statuses.size());
}
@@ -96,12 +84,8 @@ public void testListFilesWithoutFilter() throws IOException {
@Test
public void testListFilesWithFilter() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listFiles(fs, new Path(base, "a"), false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listFiles(fs, new Path(base, "a"), false,
+ (PathFilter) path -> path.getName().endsWith(".txt"));
assertEquals("File count should match", 3, statuses.size());
Collections.sort(statuses);
@@ -118,12 +102,8 @@ public void testListFilesRecursiveWithoutFilter() throws IOException {
@Test
public void testListFilesRecursiveWithFilter() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listFiles(fs, base, true, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a") || path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listFiles(fs, base, true,
+ (PathFilter) path -> path.getName().endsWith("a") || path.getName().endsWith(".txt"));
assertEquals("File count should match", 8, statuses.size());
}
@@ -142,12 +122,8 @@ public void testListAllWithoutFilter() throws IOException {
@Test
public void testListAllWithFilter() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listAll(fs, new Path(base, "a"), false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a") || path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listAll(fs, new Path(base, "a"), false,
+ (PathFilter) path -> path.getName().endsWith("a") || path.getName().endsWith(".txt"));
assertEquals("Directory and file count should match", 4, statuses.size());
}
@@ -159,34 +135,21 @@ public void testListAllRecursiveWithoutFilter() throws IOException {
@Test
public void testListAllRecursiveWithFilter() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listAll(fs, new Path(base, "a"), true, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith("a") || path.getName().endsWith(".txt");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listAll(fs, new Path(base, "a"), true,
+ (PathFilter) path -> path.getName().endsWith("a") || path.getName().endsWith(".txt"));
assertEquals("Directory and file count should match", 7, statuses.size());
}
@Test
public void testListAllEmptyResult() throws IOException {
- List<FileStatus> statuses = FileSystemUtil.listAll(fs, base, false, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("xyz");
- }
- });
+ List<FileStatus> statuses = FileSystemUtil.listAll(fs, base, false,
+ (PathFilter) path -> path.getName().startsWith("xyz"));
assertEquals("Directory and file count should match", 0, statuses.size());
}
@Test
public void testMergeFiltersWithMissingParameters() {
- PathFilter filter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("a");
- }
- };
+ PathFilter filter = path -> path.getName().startsWith("a");
assertEquals("Should have returned initial filter", filter, FileSystemUtil.mergeFilters(filter, null));
assertEquals("Should have returned initial filter", filter, FileSystemUtil.mergeFilters(filter, new PathFilter[]{}));
@@ -197,19 +160,8 @@ public boolean accept(Path path) {
public void mergeFiltersTrue() {
Path file = new Path("abc.txt");
- PathFilter firstFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("a");
- }
- };
-
- PathFilter secondFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith(".txt");
- }
- };
+ PathFilter firstFilter = path -> path.getName().startsWith("a");
+ PathFilter secondFilter = path -> path.getName().endsWith(".txt");
assertTrue("Path should have been included in the path list", FileSystemUtil.mergeFilters(firstFilter, secondFilter).accept(file));
assertTrue("Path should have been included in the path list", FileSystemUtil.mergeFilters(firstFilter, new PathFilter[] {secondFilter}).accept(file));
@@ -219,22 +171,32 @@ public boolean accept(Path path) {
public void mergeFiltersFalse() {
Path file = new Path("abc.txt");
- PathFilter firstFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("a");
- }
- };
-
- PathFilter secondFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith(".csv");
- }
- };
+ PathFilter firstFilter = path -> path.getName().startsWith("a");
+ PathFilter secondFilter = path -> path.getName().endsWith(".csv");
assertFalse("Path should have been excluded from the path list", FileSystemUtil.mergeFilters(firstFilter, secondFilter).accept(file));
assertFalse("Path should have been excluded from the path list", FileSystemUtil.mergeFilters(firstFilter, new PathFilter[] {secondFilter}).accept(file));
}
+ @Test
+ public void testListDirectoriesSafe() {
+ Path file = new Path(base, "missing");
+ List<FileStatus> fileStatuses = FileSystemUtil.listDirectoriesSafe(fs, file, true);
+ assertTrue("Should return empty result", fileStatuses.isEmpty());
+ }
+
+ @Test
+ public void testListFilesSafe() {
+ Path file = new Path(base, "missing.txt");
+ List<FileStatus> fileStatuses = FileSystemUtil.listFilesSafe(fs, file, true);
+ assertTrue("Should return empty result", fileStatuses.isEmpty());
+ }
+
+ @Test
+ public void testListAllSafe() {
+ Path file = new Path(base, "missing");
+ List<FileStatus> fileStatuses = FileSystemUtil.listAllSafe(fs, file, true);
+ assertTrue("Should return empty result", fileStatuses.isEmpty());
+ }
+
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTestBase.java
index 22cf3ea7b14..5081bfd83ac 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/FileSystemUtilTestBase.java
@@ -68,12 +68,7 @@ public static void setup() throws IOException {
// create temporary directory with sub-folders and files
final File tempDir = Files.createTempDir();
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- FileUtils.deleteQuietly(tempDir);
- }
- });
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> FileUtils.deleteQuietly(tempDir)));
base = new Path(tempDir.toURI().getPath());
createDefaultStructure(fs, base, "a", 2);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services