You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/01 00:59:17 UTC
hadoop git commit: HDFS-7707. Edit log corruption due to delayed
block removal again. Contributed by Yongjun Zhang (cherry picked from commit
843806d03ab1a24f191782f42eb817505228eb9f)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 e917ef3cf -> a1c3e6a05
HDFS-7707. Edit log corruption due to delayed block removal again. Contributed by Yongjun Zhang
(cherry picked from commit 843806d03ab1a24f191782f42eb817505228eb9f)
(cherry picked from commit c428d303f67bef3a7df12153947c6b0199a0938b)
(cherry picked from commit 21d8b22feef2fc9f6793d9c157391b142368e588)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1c3e6a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1c3e6a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1c3e6a0
Branch: refs/heads/branch-2.6.1
Commit: a1c3e6a058e4041ce7d1601867154b9ddd0d9c30
Parents: e917ef3
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Feb 3 14:45:15 2015 -0600
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Aug 31 15:55:07 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 14 +++++++--
.../TestCommitBlockSynchronization.java | 4 ++-
.../hdfs/server/namenode/TestDeleteRace.java | 32 +++++++++++++++-----
4 files changed, 43 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ec049b0..648f1ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -67,6 +67,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7575. Upgrade should generate a unique storage ID for each
volume. (Arpit Agarwal)
+ HDFS-7707. Edit log corruption due to delayed block removal again.
+ (Yongjun Zhang via kihwal)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7077b68..8e5a2db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -238,6 +238,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -6864,13 +6865,22 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
INode tmpChild = file;
INodeDirectory tmpParent = file.getParent();
while (true) {
- if (tmpParent == null ||
- tmpParent.searchChildren(tmpChild.getLocalNameBytes()) < 0) {
+ if (tmpParent == null) {
return true;
}
+
+ INode childINode = tmpParent.getChild(tmpChild.getLocalNameBytes(),
+ Snapshot.CURRENT_STATE_ID);
+ if (childINode == null || !childINode.equals(tmpChild)) {
+ // a newly created INode with the same name as an already deleted one
+ // would be a different INode than the deleted one
+ return true;
+ }
+
if (tmpParent.isRoot()) {
break;
}
+
tmpChild = tmpParent;
tmpParent = tmpParent.getParent();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index bd71870..816c7c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -54,7 +54,9 @@ public class TestCommitBlockSynchronization {
// set file's parent as root and put the file to inodeMap, so
// FSNamesystem's isFileDeleted() method will return false on this file
if (file.getParent() == null) {
- INodeDirectory parent = mock(INodeDirectory.class);
+ INodeDirectory mparent = mock(INodeDirectory.class);
+ INodeDirectory parent = new INodeDirectory(mparent.getId(), new byte[0],
+ mparent.getPermissionStatus(), mparent.getAccessTime());
parent.setLocalName(new byte[0]);
parent.addChild(file);
file.setParent(parent);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 267821f..7d4eb31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.FileNotFoundException;
+import java.util.AbstractMap;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -226,10 +228,19 @@ public class TestDeleteRace {
private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot)
throws Exception {
LOG.info("Start testing, hasSnapshot: " + hasSnapshot);
- final String testPaths[] = {
- "/test-file",
- "/testdir/testdir1/test-file"
- };
+ ArrayList<AbstractMap.SimpleImmutableEntry<String, Boolean>> testList =
+ new ArrayList<AbstractMap.SimpleImmutableEntry<String, Boolean>> ();
+ testList.add(
+ new AbstractMap.SimpleImmutableEntry<String, Boolean>("/test-file", false));
+ testList.add(
+ new AbstractMap.SimpleImmutableEntry<String, Boolean>("/test-file1", true));
+ testList.add(
+ new AbstractMap.SimpleImmutableEntry<String, Boolean>(
+ "/testdir/testdir1/test-file", false));
+ testList.add(
+ new AbstractMap.SimpleImmutableEntry<String, Boolean>(
+ "/testdir/testdir1/test-file1", true));
+
final Path rootPath = new Path("/");
final Configuration conf = new Configuration();
// Disable permissions so that another user can recover the lease.
@@ -247,8 +258,11 @@ public class TestDeleteRace {
DistributedFileSystem fs = cluster.getFileSystem();
int stId = 0;
- for (String testPath : testPaths) {
- LOG.info("test on " + testPath + " snapshot: " + hasSnapshot);
+ for(AbstractMap.SimpleImmutableEntry<String, Boolean> stest : testList) {
+ String testPath = stest.getKey();
+ Boolean mkSameDir = stest.getValue();
+ LOG.info("test on " + testPath + " mkSameDir: " + mkSameDir
+ + " snapshot: " + hasSnapshot);
Path fPath = new Path(testPath);
//find grandest non-root parent
Path grandestNonRootParent = fPath;
@@ -304,7 +318,11 @@ public class TestDeleteRace {
LOG.info("Deleting recursively " + grandestNonRootParent);
fs.delete(grandestNonRootParent, true);
-
+ if (mkSameDir && !grandestNonRootParent.toString().equals(testPath)) {
+ LOG.info("Recreate dir " + grandestNonRootParent + " testpath: "
+ + testPath);
+ fs.mkdirs(grandestNonRootParent);
+ }
delayer.proceed();
LOG.info("Now wait for result");
delayer.waitForResult();