You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2025/09/14 21:15:43 UTC
(hadoop) branch branch-3.4 updated (a40f5f4de0f -> 03f83be7c16)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a change to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from a40f5f4de0f HADOOP-19685. Fix double semicolon breaking clover
new d874cf09104 HDFS-17380. FsImageValidation: remove inaccessible nodes. (#6549). Contributed by Tsz-wo Sze.
new 03f83be7c16 HDFS-17528. FsImageValidation: set txid when saving a new image (#6828)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hdfs/server/namenode/FSImage.java | 18 +++++-
.../hdfs/server/namenode/FsImageValidation.java | 75 +++++++++++++++++-----
2 files changed, 75 insertions(+), 18 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/02: HDFS-17380. FsImageValidation: remove inaccessible nodes. (#6549). Contributed by Tsz-wo Sze.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d874cf09104f5f30f0f8e9109dc020b245ebb9de
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Sun Mar 17 20:52:17 2024 -0700
HDFS-17380. FsImageValidation: remove inaccessible nodes. (#6549). Contributed by Tsz-wo Sze.
Signed-off-by: He Xiaoqiao <he...@apache.org>
(cherry picked from commit b25b28e5bbda19eb24fcfbd8fb5e61acfb343ff4)
---
.../hadoop/hdfs/server/namenode/FSImage.java | 9 +++
.../hdfs/server/namenode/FsImageValidation.java | 68 +++++++++++++++++-----
2 files changed, 63 insertions(+), 14 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 12f79e8c52e..187e253e420 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -24,6 +24,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -1213,6 +1214,14 @@ public void removeFromCheckpointing(long txid) {
currentlyCheckpointing.remove(txid);
}
+ void save(FSNamesystem src, File dst) throws IOException {
+ final SaveNamespaceContext context = new SaveNamespaceContext(src,
+ getCorrectLastAppliedOrWrittenTxId(), new Canceler());
+ final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst);
+ Files.createDirectories(storageDirectory.getCurrentDir().toPath());
+ new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run();
+ }
+
private synchronized void saveFSImageInAllDirs(FSNamesystem source,
NameNodeFile nnf, long txid, Canceler canceler) throws IOException {
StartupProgress prog = NameNode.getStartupProgress();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
index ab301104f2e..275be905051 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
@@ -43,6 +43,8 @@
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
@@ -77,6 +79,35 @@ public class FsImageValidation {
static final String FS_IMAGE = "FS_IMAGE";
+ /**
+ * Use an environment variable "PRINT_ERROR" to enable/disable printing error messages.
+ * The default is true
+ */
+ static final boolean PRINT_ERROR;
+
+ static {
+ PRINT_ERROR = getEnvBoolean("PRINT_ERROR", true);
+ }
+
+ /**
+ * @return the boolean value of an environment property.
+ * If the environment property is not set or cannot be parsed as a boolean,
+ * return the default value.
+ */
+ static boolean getEnvBoolean(String property, boolean defaultValue) {
+ final String env = System.getenv().get(property);
+ final boolean setToNonDefault = ("" + !defaultValue).equalsIgnoreCase(env);
+ // default | setToNonDefault | value
+ // ---------------------------------
+ // true | true | false
+ // true | false | true
+ // false | true | true
+ // false | false | false
+ final boolean value = defaultValue != setToNonDefault;
+ LOG.info("ENV: {} = {} (\"{}\")", property, value, env);
+ return value;
+ }
+
static String getEnv(String property) {
final String value = System.getenv().get(property);
LOG.info("ENV: {} = {}", property, value);
@@ -186,13 +217,19 @@ int run(Configuration conf, AtomicInteger errorCount) throws Exception {
final FSNamesystem namesystem = checkINodeReference(conf, errorCount);
// check INodeMap
- INodeMapValidation.run(namesystem.getFSDirectory(), errorCount);
+ final boolean changed = INodeMapValidation.run(namesystem.getFSDirectory(), errorCount);
LOG.info(Util.memoryInfo());
final int d = errorCount.get() - initCount;
if (d > 0) {
Cli.println("Found %d error(s) in %s", d, fsImageFile.getAbsolutePath());
}
+ if (changed) {
+ final File dir = fsImageFile.isDirectory()? fsImageFile: fsImageFile.getParentFile();
+ final Path temp = Files.createTempDirectory(dir.toPath(), "newFsImage");
+ Cli.println("INodeMap changed, save a new FSImage to %s", temp);
+ namesystem.getFSImage().save(namesystem, temp.toFile());
+ }
return d;
}
@@ -261,27 +298,26 @@ FSNamesystem checkINodeReference(Configuration conf,
}
static class INodeMapValidation {
- static Iterable<INodeWithAdditionalFields> iterate(INodeMap map) {
- return new Iterable<INodeWithAdditionalFields>() {
- @Override
- public Iterator<INodeWithAdditionalFields> iterator() {
- return map.getMapIterator();
- }
- };
- }
-
- static void run(FSDirectory fsdir, AtomicInteger errorCount) {
+ static boolean run(FSDirectory fsdir, AtomicInteger errorCount) {
+ final String name = INodeMapValidation.class.getSimpleName();
final int initErrorCount = errorCount.get();
final Counts counts = INodeCountVisitor.countTree(fsdir.getRoot());
- for (INodeWithAdditionalFields i : iterate(fsdir.getINodeMap())) {
+ final INodeMap map = fsdir.getINodeMap();
+ final int oldSize = map.size();
+ println("%s INodeMap old size: %d", name, oldSize);
+ for (final Iterator<INodeWithAdditionalFields> j = map.getMapIterator(); j.hasNext();) {
+ final INodeWithAdditionalFields i = j.next();
if (counts.getCount(i) == 0) {
+ j.remove();
Cli.printError(errorCount, "%s (%d) is inaccessible (%s)",
i, i.getId(), i.getFullPathName());
}
}
- println("%s ended successfully: %d error(s) found.",
- INodeMapValidation.class.getSimpleName(),
+ final int newSize = map.size();
+ println("%s INodeMap new size: %d", name, newSize);
+ println("%s ended successfully: %d error(s) found.", name,
errorCount.get() - initErrorCount);
+ return newSize != oldSize;
}
}
@@ -343,6 +379,10 @@ static synchronized void printError(String message, Throwable t) {
static synchronized void printError(AtomicInteger errorCount,
String format, Object... args) {
final int count = errorCount.incrementAndGet();
+ if (!PRINT_ERROR) {
+ return;
+ }
+
final String s = "FSIMAGE_ERROR " + count + ": "
+ String.format(format, args);
System.out.println(s);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 02/02: HDFS-17528. FsImageValidation: set txid when saving a new image (#6828)
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 03f83be7c16632d3c7bb3d796facd9b4d4477cf2
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Wed Jun 19 11:38:17 2024 +0800
HDFS-17528. FsImageValidation: set txid when saving a new image (#6828)
(cherry picked from commit 1e6411c9ec610f406f71a8d9f54d32a400c26815)
---
.../org/apache/hadoop/hdfs/server/namenode/FSImage.java | 13 +++++++++----
.../hadoop/hdfs/server/namenode/FsImageValidation.java | 7 +++++--
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 187e253e420..15096ef028e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -977,12 +977,16 @@ private void loadFSImage(File curFile, MD5Hash expectedMd5,
" but expecting " + expectedMd5);
}
- long txId = loader.getLoadedImageTxId();
+ final long txId = setLastAppliedTxId(loader);
LOG.info("Loaded image for txid " + txId + " from " + curFile);
- lastAppliedTxId = txId;
storage.setMostRecentCheckpointInfo(txId, curFile.lastModified());
}
+ synchronized long setLastAppliedTxId(FSImageFormat.LoaderDelegator loader) {
+ lastAppliedTxId = loader.getLoadedImageTxId();
+ return lastAppliedTxId;
+ }
+
/**
* Save the contents of the FS image to the file.
*/
@@ -1215,8 +1219,9 @@ public void removeFromCheckpointing(long txid) {
}
void save(FSNamesystem src, File dst) throws IOException {
- final SaveNamespaceContext context = new SaveNamespaceContext(src,
- getCorrectLastAppliedOrWrittenTxId(), new Canceler());
+ final long txid = getCorrectLastAppliedOrWrittenTxId();
+ LOG.info("save fsimage with txid={} to {}", txid, dst.getAbsolutePath());
+ final SaveNamespaceContext context = new SaveNamespaceContext(src, txid, new Canceler());
final Storage.StorageDirectory storageDirectory = new Storage.StorageDirectory(dst);
Files.createDirectories(storageDirectory.getCurrentDir().toPath());
new FSImageSaver(context, storageDirectory, NameNodeFile.IMAGE).run();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
index 275be905051..4dac221e4d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
@@ -214,6 +214,7 @@ int run(Configuration conf, AtomicInteger errorCount) throws Exception {
initConf(conf);
// check INodeReference
+ NameNode.initMetrics(conf, HdfsServerConstants.NamenodeRole.NAMENODE); // to avoid NPE
final FSNamesystem namesystem = checkINodeReference(conf, errorCount);
// check INodeMap
@@ -276,14 +277,16 @@ public void run() {
namesystem.getFSDirectory().writeLock();
try {
loader.load(fsImageFile, false);
+ fsImage.setLastAppliedTxId(loader);
} finally {
namesystem.getFSDirectory().writeUnlock();
namesystem.writeUnlock("loadImage");
}
}
t.cancel();
- Cli.println("Loaded %s %s successfully in %s",
- FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart));
+ Cli.println("Loaded %s %s with txid %d successfully in %s",
+ FS_IMAGE, fsImageFile, namesystem.getFSImage().getLastAppliedTxId(),
+ StringUtils.formatTime(now() - loadStart));
return namesystem;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org