You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yj...@apache.org on 2018/02/07 23:21:46 UTC
[1/2] hadoop git commit: HDFS-12985. NameNode crashes during restart
after an OpenForWrite file present in the Snapshot got deleted. (cherry
picked from commit 73ff09b79a5cf9932edc21c58f3a730f7379086b)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0.1 2a3cefa02 -> 60d82ac61
HDFS-12985. NameNode crashes during restart after an OpenForWrite file present in the Snapshot got deleted.
(cherry picked from commit 73ff09b79a5cf9932edc21c58f3a730f7379086b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82172990
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82172990
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82172990
Branch: refs/heads/branch-3.0.1
Commit: 821729905ed8439ee5f15372faaf6b40aff7f155
Parents: 2a3cefa
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Mon Jan 8 15:34:00 2018 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Wed Feb 7 14:20:05 2018 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INodeFile.java | 16 ++++---
.../snapshot/TestOpenFilesWithSnapshot.java | 45 ++++++++++++++++++++
2 files changed, 55 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82172990/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 906a940..90659f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -727,6 +727,13 @@ public class INodeFile extends INodeWithAdditionalFields
this.blocks = BlockInfo.EMPTY_ARRAY;
}
+ private void updateRemovedUnderConstructionFiles(
+ ReclaimContext reclaimContext) {
+ if (isUnderConstruction() && reclaimContext.removedUCFiles != null) {
+ reclaimContext.removedUCFiles.add(getId());
+ }
+ }
+
@Override
public void cleanSubtree(ReclaimContext reclaimContext,
final int snapshot, int priorSnapshotId) {
@@ -735,6 +742,7 @@ public class INodeFile extends INodeWithAdditionalFields
// TODO: avoid calling getStoragePolicyID
sf.cleanFile(reclaimContext, this, snapshot, priorSnapshotId,
getStoragePolicyID());
+ updateRemovedUnderConstructionFiles(reclaimContext);
} else {
if (snapshot == CURRENT_STATE_ID) {
if (priorSnapshotId == NO_SNAPSHOT_ID) {
@@ -747,9 +755,7 @@ public class INodeFile extends INodeWithAdditionalFields
// clean the 0-sized block if the file is UC
if (uc != null) {
uc.cleanZeroSizeBlock(this, reclaimContext.collectedBlocks);
- if (reclaimContext.removedUCFiles != null) {
- reclaimContext.removedUCFiles.add(getId());
- }
+ updateRemovedUnderConstructionFiles(reclaimContext);
}
}
}
@@ -768,9 +774,7 @@ public class INodeFile extends INodeWithAdditionalFields
reclaimContext.collectedBlocks);
sf.clearDiffs();
}
- if (isUnderConstruction() && reclaimContext.removedUCFiles != null) {
- reclaimContext.removedUCFiles.add(getId());
- }
+ updateRemovedUnderConstructionFiles(reclaimContext);
}
public void clearFile(ReclaimContext reclaimContext) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82172990/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index 537612c..a35e141 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -631,6 +631,51 @@ public class TestOpenFilesWithSnapshot {
}
/**
+ * Verify if the NameNode can restart properly after an OpenForWrite
+ * file and the only snapshot it was present in were deleted.
+ *
+ * @throws Exception
+ */
+ @Test (timeout = 600000)
+ public void testOpenFileDeletionAndNNRestart() throws Exception {
+ // Construct the directory tree
+ final Path snapRootDir = new Path("/level_0_A/test");
+ final String hbaseFileName = "hbase.log";
+ final String snap1Name = "snap_1";
+
+ // Create a file with few blocks. Get its output stream
+ // for append.
+ final Path hbaseFile = new Path(snapRootDir, hbaseFileName);
+ createFile(hbaseFile);
+ FSDataOutputStream hbaseOutputStream = fs.append(hbaseFile);
+
+ int newWriteLength = (int) (BLOCKSIZE * 1.5);
+ byte[] buf = new byte[newWriteLength];
+ Random random = new Random();
+ random.nextBytes(buf);
+
+ // Write more data to the file
+ writeToStream(hbaseOutputStream, buf);
+
+ // Take a snapshot while the file is open for write
+ final Path snap1Dir = SnapshotTestHelper.createSnapshot(
+ fs, snapRootDir, snap1Name);
+ LOG.info("Open file status in snap: " +
+ fs.getFileStatus(new Path(snap1Dir, hbaseFileName)));
+
+ // Delete the open file and the snapshot while
+ // its output stream is still open.
+ fs.delete(hbaseFile, true);
+ fs.deleteSnapshot(snapRootDir, snap1Name);
+ Assert.assertFalse(fs.exists(hbaseFile));
+
+ // Verify file existence after the NameNode restart
+ cluster.restartNameNode();
+ cluster.waitActive();
+ Assert.assertFalse(fs.exists(hbaseFile));
+ }
+
+ /**
* Test client writing to open files are not interrupted when snapshots
* that captured open files get deleted.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HDFS-13115. In
getNumUnderConstructionBlocks(),
ignore the inodeIds for which the inodes have been deleted. Contributed by
Yongjun Zhang. (cherry picked from commit
f491f717e9ee6b75ad5cfca48da9c6297e94a8f7)
Posted by yj...@apache.org.
HDFS-13115. In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted. Contributed by Yongjun Zhang.
(cherry picked from commit f491f717e9ee6b75ad5cfca48da9c6297e94a8f7)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60d82ac6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60d82ac6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60d82ac6
Branch: refs/heads/branch-3.0.1
Commit: 60d82ac61f2afa2a61c8eeabf20e45b5a89311d0
Parents: 8217299
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Wed Feb 7 14:19:18 2018 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Wed Feb 7 15:12:29 2018 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/LeaseManager.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60d82ac6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 29b4fe1..47e0891 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -142,7 +142,15 @@ public class LeaseManager {
+ "acquired before counting under construction blocks";
long numUCBlocks = 0;
for (Long id : getINodeIdWithLeases()) {
- final INodeFile cons = fsnamesystem.getFSDirectory().getInode(id).asFile();
+ INode inode = fsnamesystem.getFSDirectory().getInode(id);
+ if (inode == null) {
+ // The inode could have been deleted after getINodeIdWithLeases() is
+ // called, check here, and ignore it if so
+ LOG.warn("Failed to find inode " + id +
+ " in getNumUnderConstructionBlocks().");
+ continue;
+ }
+ final INodeFile cons = inode.asFile();
if (!cons.isUnderConstruction()) {
LOG.warn("The file " + cons.getFullPathName()
+ " is not under construction but has lease.");
@@ -153,10 +161,11 @@ public class LeaseManager {
continue;
}
for(BlockInfo b : blocks) {
- if(!b.isComplete())
+ if(!b.isComplete()) {
numUCBlocks++;
}
}
+ }
LOG.info("Number of blocks under construction: " + numUCBlocks);
return numUCBlocks;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org