You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2019/11/15 18:54:49 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-961] Bypass
locked directories when calculating src watermark
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 1b3a16e [GOBBLIN-961] Bypass locked directories when calculating src watermark
1b3a16e is described below
commit 1b3a16e1cb9e0494e3d875ca66cbd01c7ea33517
Author: Zihan Li <zi...@zihli-mn1.linkedin.biz>
AuthorDate: Fri Nov 15 10:54:42 2019 -0800
[GOBBLIN-961] Bypass locked directories when calculating src watermark
Closes #2810 from ZihanLi58/GOBBLIN-961
---
.../data/management/copy/replication/SourceHadoopFsEndPoint.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
index 2a56f2e..2137d77 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
@@ -75,8 +75,11 @@ public class SourceHadoopFsEndPoint extends HadoopFsEndPoint {
Collection<Path> validPaths = ReplicationDataValidPathPicker.getValidPaths(this);
for (Path p : validPaths) {
- this.allFileStatus.addAll(
- FileListUtils.listFilesRecursively(fs, p, super.getPathFilter(), super.isApplyFilterToDirectories()));
+ try {
+ this.allFileStatus.addAll(FileListUtils.listFilesRecursively(fs, p, super.getPathFilter(), super.isApplyFilterToDirectories()));
+ } catch (Exception e) {
+ log.error(String.format("Error while try read file in directory %s to get watermark", p.toString()));
+ }
}
for (FileStatus f : this.allFileStatus) {