You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/11/29 18:33:41 UTC
[hadoop] branch trunk updated: HDFS-15009. FSCK
-list-corruptfileblocks return Invalid Entries. Contributed by
hemanthboyina.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6b2d6d4 HDFS-15009. FSCK -list-corruptfileblocks return Invalid Entries. Contributed by hemanthboyina.
6b2d6d4 is described below
commit 6b2d6d4aafb110bef1b77d4ccbba4350e624b57d
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Fri Nov 29 23:55:37 2019 +0530
HDFS-15009. FSCK -list-corruptfileblocks return Invalid Entries. Contributed by hemanthboyina.
---
.../federation/resolver/MountTableResolver.java | 2 +-
.../server/federation/router/FederationUtil.java | 20 --------------------
.../hadoop/hdfs/server/federation/router/Quota.java | 4 +++-
.../server/federation/router/RouterQuotaManager.java | 2 +-
.../federation/store/impl/MountTableStoreImpl.java | 5 +++--
.../main/java/org/apache/hadoop/hdfs/DFSUtil.java | 18 ++++++++++++++++++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 3 ++-
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 3 ++-
.../apache/hadoop/hdfs/server/namenode/TestFsck.java | 8 +++++++-
9 files changed, 37 insertions(+), 28 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index 6becedc..96b560c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -24,7 +24,7 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDE
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT;
-import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
+import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry;
import java.io.IOException;
import java.util.Collection;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index df22c4a..d2eec04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -27,7 +27,6 @@ import java.net.URLConnection;
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -225,25 +224,6 @@ public final class FederationUtil {
}
/**
- * Check if the given path is the child of parent path.
- * @param path Path to be check.
- * @param parent Parent path.
- * @return True if parent path is parent entry for given path.
- */
- public static boolean isParentEntry(final String path, final String parent) {
- if (!path.startsWith(parent)) {
- return false;
- }
-
- if (path.equals(parent)) {
- return true;
- }
-
- return path.charAt(parent.length()) == Path.SEPARATOR_CHAR
- || parent.equals(Path.SEPARATOR);
- }
-
- /**
* Add the the number of children for an existing HdfsFileStatus object.
* @param dirStatus HdfsfileStatus object.
* @param children number of children to be added.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
index ee65608..0a08f72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -217,7 +219,7 @@ public class Quota {
boolean isChildPath = false;
for (RemoteLocation d : dests) {
- if (FederationUtil.isParentEntry(loc.getDest(), d.getDest())) {
+ if (isParentEntry(loc.getDest(), d.getDest())) {
isChildPath = true;
break;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
index f061cc5..c1a5146 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
-import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
+import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry;
import java.util.HashSet;
import java.util.Map.Entry;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
index 8682a3d..fb9f6a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.federation.store.impl;
+import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry;
+
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
@@ -25,7 +27,6 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
@@ -140,7 +141,7 @@ public class MountTableStoreImpl extends MountTableStore {
while (it.hasNext()) {
MountTable record = it.next();
String srcPath = record.getSourcePath();
- if (!FederationUtil.isParentEntry(srcPath, reqSrcPath)) {
+ if (!isParentEntry(srcPath, reqSrcPath)) {
it.remove();
} else if (pc != null) {
// do the READ permission check
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 8ad11f8..1968e32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -1816,4 +1816,22 @@ public class DFSUtil {
return flags;
}
+ /**
+ * Check if the given path is the child of parent path.
+ * @param path Path to be check.
+ * @param parent Parent path.
+ * @return True if parent path is parent entry for given path.
+ */
+ public static boolean isParentEntry(final String path, final String parent) {
+ if (!path.startsWith(parent)) {
+ return false;
+ }
+
+ if (path.equals(parent)) {
+ return true;
+ }
+
+ return path.charAt(parent.length()) == Path.SEPARATOR_CHAR
+ || parent.equals(Path.SEPARATOR);
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0bf2166..8d1884e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -89,6 +89,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
@@ -5665,7 +5666,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
skip++;
if (inode != null) {
String src = inode.getFullPathName();
- if (src.startsWith(path)){
+ if (isParentEntry(src, path)) {
corruptFiles.add(new CorruptFileBlockInfo(src, blk));
count++;
if (count >= maxCorruptFileBlocksReturn)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index ad8aaaa..69b1555 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -94,7 +94,8 @@ import com.google.common.annotations.VisibleForTesting;
* This class provides rudimentary checking of DFS volumes for errors and
* sub-optimal conditions.
* <p>The tool scans all files and directories, starting from an indicated
- * root path. The following abnormal conditions are detected and handled:</p>
+ * root path and its descendants. The following abnormal conditions are
+ * detected and handled:</p>
* <ul>
* <li>files with blocks that are completely missing from all datanodes.<br>
* In this case the tool can perform one of the following actions:
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index a37ed2d..e4aed89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -1171,8 +1171,14 @@ public class TestFsck {
outStr = runFsck(conf, 0, true, "/goodData", "-list-corruptfileblocks");
System.out.println("3. good fsck out: " + outStr);
assertTrue(outStr.contains("has 0 CORRUPT files"));
- util.cleanup(fs, "/corruptData");
util.cleanup(fs, "/goodData");
+
+ // validate if a directory have any invalid entries
+ util.createFiles(fs, "/corruptDa");
+ outStr = runFsck(conf, 0, true, "/corruptDa", "-list-corruptfileblocks");
+ assertTrue(outStr.contains("has 0 CORRUPT files"));
+ util.cleanup(fs, "/corruptData");
+ util.cleanup(fs, "/corruptDa");
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org