You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by zi...@apache.org on 2021/12/20 20:39:31 UTC
[gobblin] branch master updated: [GOBBLIN-1592] Make hive copy be able to apply filter on directory (#3446)
This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 6e5ba1f [GOBBLIN-1592] Make hive copy be able to apply filter on directory (#3446)
6e5ba1f is described below
commit 6e5ba1fcd57167444f3a2ce876fcd43eee3b3d98
Author: Zihan Li <zi...@linkedin.com>
AuthorDate: Mon Dec 20 12:39:22 2021 -0800
[GOBBLIN-1592] Make hive copy be able to apply filter on directory (#3446)
---
.../org/apache/gobblin/data/management/copy/RecursivePathFinder.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/RecursivePathFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/RecursivePathFinder.java
index b749996..11c0e18 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/RecursivePathFinder.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/RecursivePathFinder.java
@@ -47,6 +47,7 @@ public class RecursivePathFinder {
private final FileSystem fs;
private final PathFilter pathFilter;
private final boolean includeEmptyDirectories;
+ private final boolean applyFilterToDirectories;
public RecursivePathFinder(final FileSystem fs, Path rootPath, Properties properties) {
this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath);
@@ -55,6 +56,7 @@ public class RecursivePathFinder {
this.pathFilter = DatasetUtils.instantiatePathFilter(properties);
this.includeEmptyDirectories =
Boolean.parseBoolean(properties.getProperty(CopyConfiguration.INCLUDE_EMPTY_DIRECTORIES));
+ this.applyFilterToDirectories = Boolean.parseBoolean(properties.getProperty(CopyConfiguration.APPLY_FILTER_TO_DIRECTORIES));
}
public Set<FileStatus> getPaths(boolean skipHiddenPaths)
@@ -66,7 +68,7 @@ public class RecursivePathFinder {
PathFilter actualFilter =
skipHiddenPaths ? new AndPathFilter(new HiddenFilter(), this.pathFilter) : this.pathFilter;
List<FileStatus> files =
- FileListUtils.listFilesToCopyAtPath(this.fs, this.rootPath, actualFilter, includeEmptyDirectories);
+ FileListUtils.listFilesToCopyAtPath(this.fs, this.rootPath, actualFilter, this.applyFilterToDirectories, includeEmptyDirectories);
return Sets.newHashSet(files);
}