You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/09/20 12:07:23 UTC
[1/4] hadoop git commit: HADOOP-15748. S3 listing inconsistency can
raise NPE in globber. Contributed by Steve Loughran.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0dd686114 -> 5b7799154
refs/heads/branch-3.0 d2655ea60 -> fc6d851f7
refs/heads/branch-3.1 2aa385463 -> 9dc180d14
refs/heads/trunk 7ad27e97f -> 646874c32
HADOOP-15748. S3 listing inconsistency can raise NPE in globber.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/646874c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/646874c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/646874c3
Branch: refs/heads/trunk
Commit: 646874c326139457b79cf8cfa547b3c91a78c7b4
Parents: 7ad27e9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 20 13:04:52 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 20 13:04:52 2018 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/Globber.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/646874c3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index ca3db1d..b241a94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -245,7 +245,18 @@ class Globber {
// incorrectly conclude that /a/b was a file and should not match
// /a/*/*. So we use getFileStatus of the path we just listed to
// disambiguate.
- if (!getFileStatus(candidate.getPath()).isDirectory()) {
+ Path path = candidate.getPath();
+ FileStatus status = getFileStatus(path);
+ if (status == null) {
+ // null means the file was not found
+ LOG.warn("File/directory {} not found:"
+ + " it may have been deleted."
+ + " If this is an object store, this can be a sign of"
+ + " eventual consistency problems.",
+ path);
+ continue;
+ }
+ if (!status.isDirectory()) {
continue;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/4] hadoop git commit: HADOOP-15748. S3 listing inconsistency can
raise NPE in globber. Contributed by Steve Loughran.
Posted by st...@apache.org.
HADOOP-15748. S3 listing inconsistency can raise NPE in globber.
Contributed by Steve Loughran.
(cherry picked from commit 646874c326139457b79cf8cfa547b3c91a78c7b4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9dc180d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9dc180d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9dc180d1
Branch: refs/heads/branch-3.1
Commit: 9dc180d14a265b49bdfb5f6a4cbdf476661e6b2d
Parents: 2aa3854
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 20 13:05:46 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 20 13:05:46 2018 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/Globber.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9dc180d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index ca3db1d..b241a94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -245,7 +245,18 @@ class Globber {
// incorrectly conclude that /a/b was a file and should not match
// /a/*/*. So we use getFileStatus of the path we just listed to
// disambiguate.
- if (!getFileStatus(candidate.getPath()).isDirectory()) {
+ Path path = candidate.getPath();
+ FileStatus status = getFileStatus(path);
+ if (status == null) {
+ // null means the file was not found
+ LOG.warn("File/directory {} not found:"
+ + " it may have been deleted."
+ + " If this is an object store, this can be a sign of"
+ + " eventual consistency problems.",
+ path);
+ continue;
+ }
+ if (!status.isDirectory()) {
continue;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/4] hadoop git commit: HADOOP-15748. S3 listing inconsistency can
raise NPE in globber. Contributed by Steve Loughran.
Posted by st...@apache.org.
HADOOP-15748. S3 listing inconsistency can raise NPE in globber.
Contributed by Steve Loughran.
(cherry picked from commit 646874c326139457b79cf8cfa547b3c91a78c7b4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc6d851f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc6d851f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc6d851f
Branch: refs/heads/branch-3.0
Commit: fc6d851f70a54aa70808590f3811cc8764615f88
Parents: d2655ea
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 20 13:06:20 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 20 13:06:20 2018 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/Globber.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc6d851f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index ca3db1d..b241a94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -245,7 +245,18 @@ class Globber {
// incorrectly conclude that /a/b was a file and should not match
// /a/*/*. So we use getFileStatus of the path we just listed to
// disambiguate.
- if (!getFileStatus(candidate.getPath()).isDirectory()) {
+ Path path = candidate.getPath();
+ FileStatus status = getFileStatus(path);
+ if (status == null) {
+ // null means the file was not found
+ LOG.warn("File/directory {} not found:"
+ + " it may have been deleted."
+ + " If this is an object store, this can be a sign of"
+ + " eventual consistency problems.",
+ path);
+ continue;
+ }
+ if (!status.isDirectory()) {
continue;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/4] hadoop git commit: HADOOP-15748. S3 listing inconsistency can
raise NPE in globber. Contributed by Steve Loughran.
Posted by st...@apache.org.
HADOOP-15748. S3 listing inconsistency can raise NPE in globber.
Contributed by Steve Loughran.
(cherry picked from commit 646874c326139457b79cf8cfa547b3c91a78c7b4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b779915
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b779915
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b779915
Branch: refs/heads/branch-2
Commit: 5b77991548c8d1b01ff6b32c17c7f4b6a852fff9
Parents: 0dd6861
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 20 13:07:07 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 20 13:07:07 2018 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/Globber.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b779915/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index ca3db1d..b241a94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -245,7 +245,18 @@ class Globber {
// incorrectly conclude that /a/b was a file and should not match
// /a/*/*. So we use getFileStatus of the path we just listed to
// disambiguate.
- if (!getFileStatus(candidate.getPath()).isDirectory()) {
+ Path path = candidate.getPath();
+ FileStatus status = getFileStatus(path);
+ if (status == null) {
+ // null means the file was not found
+ LOG.warn("File/directory {} not found:"
+ + " it may have been deleted."
+ + " If this is an object store, this can be a sign of"
+ + " eventual consistency problems.",
+ path);
+ continue;
+ }
+ if (!status.isDirectory()) {
continue;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org