You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2025/04/17 00:37:14 UTC
(hadoop) branch branch-3.3 updated: HDFS-17768. Observer namenode network delay causing empty block location for getBatchedListing (#7593)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 448bbe1b963 HDFS-17768. Observer namenode network delay causing empty block location for getBatchedListing (#7593)
448bbe1b963 is described below
commit 448bbe1b963ae4aa6b9475a9b33f482e39059250
Author: Dimas Shidqi Parikesit <di...@gmail.com>
AuthorDate: Wed Apr 16 20:27:43 2025 -0400
HDFS-17768. Observer namenode network delay causing empty block location for getBatchedListing (#7593)
(cherry picked from commit caf8af0de3e63ba8423760582bc640d51f0d558b)
(cherry picked from commit 8eb2211b3ffa3e3b487093eee65a77d4c115737f)
---
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 13 +++++++++++++
.../hadoop/hdfs/server/namenode/ha/TestObserverNode.java | 3 +++
2 files changed, 16 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 5ffe50caa61..0d0cfb4a501 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4188,11 +4188,22 @@ BatchedDirectoryListing getBatchedListing(String[] srcs, byte[] startAfter,
if (dirListing == null) {
throw new FileNotFoundException("Path " + src + " does not exist");
}
+ if (needLocation && isObserver()) {
+ for (HdfsFileStatus fs : dirListing.getPartialListing()) {
+ if (fs instanceof HdfsLocatedFileStatus) {
+ LocatedBlocks lbs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks();
+ checkBlockLocationsWhenObserver(lbs, fs.toString());
+ }
+ }
+ }
listing = new HdfsPartialListing(
srcsIndex, Lists.newArrayList(dirListing.getPartialListing()));
numEntries += listing.getPartialListing().size();
lastListing = dirListing;
} catch (Exception e) {
+ if (e instanceof ObserverRetryOnActiveException) {
+ throw (ObserverRetryOnActiveException) e;
+ }
if (e instanceof AccessControlException) {
logAuditEvent(false, operationName, src);
}
@@ -4255,6 +4266,8 @@ BatchedDirectoryListing getBatchedListing(String[] srcs, byte[] startAfter,
true,
returnedStartAfter);
}
+ } catch(ObserverRetryOnActiveException e){
+ throw e;
} finally {
readUnlock(operationName);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
index bb901423bb5..39be61f6dec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
@@ -376,6 +376,9 @@ public void testObserverNodeBlockMissingRetry() throws Exception {
dfs.getClient().getLocatedFileInfo(testPath.toString(), false);
assertSentTo(0);
+ dfs.getClient().batchedListPaths(new String[]{"/"}, new byte[0], true);
+ assertSentTo(0);
+
Mockito.reset(bmSpy);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org