You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2017/12/01 03:23:10 UTC
[1/5] hadoop git commit: HDFS-12638. Delete copy-on-truncate block
along with the original block,
when deleting a file being truncated. Contributed by Konstantin Shvachko.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 7252e18c3 -> 85eb32b12
refs/heads/branch-2.8 eacccf193 -> 19c18f7cc
refs/heads/branch-2.9 5a8a1e66c -> 0f5ec01ef
refs/heads/branch-3.0 58d849b85 -> def87db9c
refs/heads/trunk a63d19d36 -> 60fd0d7fd
HDFS-12638. Delete copy-on-truncate block along with the original block, when deleting a file being truncated. Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60fd0d7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60fd0d7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60fd0d7f
Branch: refs/heads/trunk
Commit: 60fd0d7fd73198fd610e59d1a4cd007c5fcc7205
Parents: a63d19d
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Nov 30 18:18:09 2017 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Nov 30 18:18:28 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INode.java | 14 +++++++
.../hdfs/server/namenode/TestFileTruncate.java | 41 ++++++++++++++++++++
2 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fd0d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 34bfe10..1682a30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -33,9 +33,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
@@ -1058,6 +1060,18 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
assert toDelete != null : "toDelete is null";
toDelete.delete();
toDeleteList.add(toDelete);
+ // If the file is being truncated
+ // the copy-on-truncate block should also be collected for deletion
+ BlockUnderConstructionFeature uc = toDelete.getUnderConstructionFeature();
+ if(uc == null) {
+ return;
+ }
+ Block truncateBlock = uc.getTruncateBlock();
+ if(truncateBlock == null || truncateBlock.equals(toDelete)) {
+ return;
+ }
+ assert truncateBlock instanceof BlockInfo : "should be BlockInfo";
+ addDeleteBlock((BlockInfo) truncateBlock);
}
public void addUpdateReplicationFactor(BlockInfo block, short targetRepl) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fd0d7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index d4215e8..51a94e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -1155,6 +1156,46 @@ public class TestFileTruncate {
fs.delete(parent, true);
}
+ /**
+ * While rolling upgrade is in-progress the test truncates a file
+ * such that copy-on-truncate is triggered, then deletes the file,
+ * and makes sure that no blocks involved in truncate are hanging around.
+ */
+ @Test
+ public void testTruncateWithRollingUpgrade() throws Exception {
+ final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0));
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ //start rolling upgrade
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"});
+ assertEquals("could not prepare for rolling upgrade", 0, status);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ Path dir = new Path("/testTruncateWithRollingUpgrade");
+ fs.mkdirs(dir);
+ final Path p = new Path(dir, "file");
+ final byte[] data = new byte[3];
+ ThreadLocalRandom.current().nextBytes(data);
+ writeContents(data, data.length, p);
+
+ assertEquals("block num should 1", 1,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+
+ final boolean isReady = fs.truncate(p, 2);
+ assertFalse("should be copy-on-truncate", isReady);
+ assertEquals("block num should 2", 2,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ fs.delete(p, true);
+
+ assertEquals("block num should 0", 0,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"});
+ assertEquals("could not finalize rolling upgrade", 0, status);
+ }
+
static void writeContents(byte[] contents, int fileLength, Path p)
throws IOException {
FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/5] hadoop git commit: HDFS-12638. Delete copy-on-truncate block
along with the original block,
when deleting a file being truncated. Contributed by Konstantin Shvachko.
Posted by sh...@apache.org.
HDFS-12638. Delete copy-on-truncate block along with the original block, when deleting a file being truncated. Contributed by Konstantin Shvachko.
(cherry picked from commit 60fd0d7fd73198fd610e59d1a4cd007c5fcc7205)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f5ec01e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f5ec01e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f5ec01e
Branch: refs/heads/branch-2.9
Commit: 0f5ec01ef7b79059197589e3651deaeb93f37ec6
Parents: 5a8a1e6
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Nov 30 18:18:09 2017 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Nov 30 18:28:18 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INode.java | 14 +++++++
.../hdfs/server/namenode/TestFileTruncate.java | 41 ++++++++++++++++++++
2 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f5ec01e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 91b1154..bca0120 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -33,9 +33,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
@@ -1057,6 +1059,18 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
assert toDelete != null : "toDelete is null";
toDelete.delete();
toDeleteList.add(toDelete);
+ // If the file is being truncated
+ // the copy-on-truncate block should also be collected for deletion
+ BlockUnderConstructionFeature uc = toDelete.getUnderConstructionFeature();
+ if(uc == null) {
+ return;
+ }
+ Block truncateBlock = uc.getTruncateBlock();
+ if(truncateBlock == null || truncateBlock.equals(toDelete)) {
+ return;
+ }
+ assert truncateBlock instanceof BlockInfo : "should be BlockInfo";
+ addDeleteBlock((BlockInfo) truncateBlock);
}
public void addUpdateReplicationFactor(BlockInfo block, short targetRepl) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f5ec01e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index e441952..7e3e1d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -1153,6 +1154,46 @@ public class TestFileTruncate {
fs.delete(parent, true);
}
+ /**
+ * While rolling upgrade is in-progress the test truncates a file
+ * such that copy-on-truncate is triggered, then deletes the file,
+ * and makes sure that no blocks involved in truncate are hanging around.
+ */
+ @Test
+ public void testTruncateWithRollingUpgrade() throws Exception {
+ final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0));
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ //start rolling upgrade
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"});
+ assertEquals("could not prepare for rolling upgrade", 0, status);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ Path dir = new Path("/testTruncateWithRollingUpgrade");
+ fs.mkdirs(dir);
+ final Path p = new Path(dir, "file");
+ final byte[] data = new byte[3];
+ ThreadLocalRandom.current().nextBytes(data);
+ writeContents(data, data.length, p);
+
+ assertEquals("block num should 1", 1,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+
+ final boolean isReady = fs.truncate(p, 2);
+ assertFalse("should be copy-on-truncate", isReady);
+ assertEquals("block num should 2", 2,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ fs.delete(p, true);
+
+ assertEquals("block num should 0", 0,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"});
+ assertEquals("could not finalize rolling upgrade", 0, status);
+ }
+
static void writeContents(byte[] contents, int fileLength, Path p)
throws IOException {
FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[5/5] hadoop git commit: HDFS-12638. Delete copy-on-truncate block
along with the original block,
when deleting a file being truncated. Contributed by Konstantin Shvachko.
Posted by sh...@apache.org.
HDFS-12638. Delete copy-on-truncate block along with the original block, when deleting a file being truncated. Contributed by Konstantin Shvachko.
(cherry picked from commit 60fd0d7fd73198fd610e59d1a4cd007c5fcc7205)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85eb32b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85eb32b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85eb32b1
Branch: refs/heads/branch-2.7
Commit: 85eb32b12d2ce7285433c77ea131dc7a964d6ef9
Parents: 7252e18
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Nov 30 18:18:09 2017 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Nov 30 19:22:01 2017 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++
.../hadoop/hdfs/server/namenode/INode.java | 17 +++++++-
.../hdfs/server/namenode/TestFileTruncate.java | 42 ++++++++++++++++++++
3 files changed, 64 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85eb32b1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 207611d..156a354 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -48,6 +48,12 @@ Release 2.7.5 - UNRELEASED
HDFS-9003. ForkJoin thread pool leaks. (Kihwal Lee via jing9)
+ HDFS-12832. INode.getFullPathName may throw ArrayIndexOutOfBoundsException
+ lead to NameNode exit. (shv)
+
+ HDFS-12638. Delete copy-on-truncate block along with the original block,
+ when deleting a file being truncated. (shv)
+
Release 2.7.4 - 2017-08-04
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85eb32b1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index e16fc7d..b23b989 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -869,9 +871,22 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* {@link BlocksMapUpdateInfo#toDeleteList}
* @param toDelete the to-be-deleted block
*/
- public void addDeleteBlock(Block toDelete) {
+ public void addDeleteBlock(BlockInfoContiguous toDelete) {
assert toDelete != null : "toDelete is null";
toDeleteList.add(toDelete);
+ // If the file is being truncated
+ // the copy-on-truncate block should also be collected for deletion
+ if(!(toDelete instanceof BlockInfoContiguousUnderConstruction)) {
+ return;
+ }
+ Block truncateBlock =
+ ((BlockInfoContiguousUnderConstruction)toDelete).getTruncateBlock();
+ if(truncateBlock == null || truncateBlock.equals(toDelete)) {
+ return;
+ }
+ assert truncateBlock instanceof BlockInfoContiguous :
+ "should be BlockInfoContiguous";
+ addDeleteBlock((BlockInfoContiguous) truncateBlock);
}
public void removeDeleteBlock(Block block) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85eb32b1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index fbcc73f..74586cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -31,6 +31,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -58,6 +59,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -1163,6 +1165,46 @@ public class TestFileTruncate {
fs.delete(parent, true);
}
+ /**
+ * While rolling upgrade is in-progress the test truncates a file
+ * such that copy-on-truncate is triggered, then deletes the file,
+ * and makes sure that no blocks involved in truncate are hanging around.
+ */
+ @Test
+ public void testTruncateWithRollingUpgrade() throws Exception {
+ final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0));
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ //start rolling upgrade
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"});
+ assertEquals("could not prepare for rolling upgrade", 0, status);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ Path dir = new Path("/testTruncateWithRollingUpgrade");
+ fs.mkdirs(dir);
+ final Path p = new Path(dir, "file");
+ final byte[] data = new byte[3];
+ ThreadLocalRandom.current().nextBytes(data);
+ writeContents(data, data.length, p);
+
+ assertEquals("block num should 1", 1,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+
+ final boolean isReady = fs.truncate(p, 2);
+ assertFalse("should be copy-on-truncate", isReady);
+ assertEquals("block num should 2", 2,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ fs.delete(p, true);
+
+ assertEquals("block num should 0", 0,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"});
+ assertEquals("could not finalize rolling upgrade", 0, status);
+ }
+
static void writeContents(byte[] contents, int fileLength, Path p)
throws IOException {
FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/5] hadoop git commit: HDFS-12638. Delete copy-on-truncate block
along with the original block,
when deleting a file being truncated. Contributed by Konstantin Shvachko.
Posted by sh...@apache.org.
HDFS-12638. Delete copy-on-truncate block along with the original block, when deleting a file being truncated. Contributed by Konstantin Shvachko.
(cherry picked from commit 60fd0d7fd73198fd610e59d1a4cd007c5fcc7205)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19c18f7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19c18f7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19c18f7c
Branch: refs/heads/branch-2.8
Commit: 19c18f7cc90c8243af634600c4c69ec1109fcce6
Parents: eacccf1
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Nov 30 18:18:09 2017 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Nov 30 18:30:11 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INode.java | 14 +++++++
.../hdfs/server/namenode/TestFileTruncate.java | 41 ++++++++++++++++++++
2 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c18f7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 779ae13..838497c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -33,9 +33,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
@@ -1051,6 +1053,18 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
assert toDelete != null : "toDelete is null";
toDelete.delete();
toDeleteList.add(toDelete);
+ // If the file is being truncated
+ // the copy-on-truncate block should also be collected for deletion
+ BlockUnderConstructionFeature uc = toDelete.getUnderConstructionFeature();
+ if(uc == null) {
+ return;
+ }
+ Block truncateBlock = uc.getTruncateBlock();
+ if(truncateBlock == null || truncateBlock.equals(toDelete)) {
+ return;
+ }
+ assert truncateBlock instanceof BlockInfo : "should be BlockInfo";
+ addDeleteBlock((BlockInfo) truncateBlock);
}
public void addUpdateReplicationFactor(BlockInfo block, short targetRepl) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19c18f7c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index f97939a..292eefd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -1153,6 +1154,46 @@ public class TestFileTruncate {
fs.delete(parent, true);
}
+ /**
+ * While rolling upgrade is in-progress the test truncates a file
+ * such that copy-on-truncate is triggered, then deletes the file,
+ * and makes sure that no blocks involved in truncate are hanging around.
+ */
+ @Test
+ public void testTruncateWithRollingUpgrade() throws Exception {
+ final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0));
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ //start rolling upgrade
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"});
+ assertEquals("could not prepare for rolling upgrade", 0, status);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ Path dir = new Path("/testTruncateWithRollingUpgrade");
+ fs.mkdirs(dir);
+ final Path p = new Path(dir, "file");
+ final byte[] data = new byte[3];
+ ThreadLocalRandom.current().nextBytes(data);
+ writeContents(data, data.length, p);
+
+ assertEquals("block num should 1", 1,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+
+ final boolean isReady = fs.truncate(p, 2);
+ assertFalse("should be copy-on-truncate", isReady);
+ assertEquals("block num should 2", 2,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ fs.delete(p, true);
+
+ assertEquals("block num should 0", 0,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"});
+ assertEquals("could not finalize rolling upgrade", 0, status);
+ }
+
static void writeContents(byte[] contents, int fileLength, Path p)
throws IOException {
FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/5] hadoop git commit: HDFS-12638. Delete copy-on-truncate block
along with the original block,
when deleting a file being truncated. Contributed by Konstantin Shvachko.
Posted by sh...@apache.org.
HDFS-12638. Delete copy-on-truncate block along with the original block, when deleting a file being truncated. Contributed by Konstantin Shvachko.
(cherry picked from commit 60fd0d7fd73198fd610e59d1a4cd007c5fcc7205)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/def87db9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/def87db9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/def87db9
Branch: refs/heads/branch-3.0
Commit: def87db9cbe52f8e89213638db00c1e0bfc7d39c
Parents: 58d849b
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Nov 30 18:18:09 2017 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Nov 30 18:27:16 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INode.java | 14 +++++++
.../hdfs/server/namenode/TestFileTruncate.java | 41 ++++++++++++++++++++
2 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/def87db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 34bfe10..1682a30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -33,9 +33,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
@@ -1058,6 +1060,18 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
assert toDelete != null : "toDelete is null";
toDelete.delete();
toDeleteList.add(toDelete);
+ // If the file is being truncated
+ // the copy-on-truncate block should also be collected for deletion
+ BlockUnderConstructionFeature uc = toDelete.getUnderConstructionFeature();
+ if(uc == null) {
+ return;
+ }
+ Block truncateBlock = uc.getTruncateBlock();
+ if(truncateBlock == null || truncateBlock.equals(toDelete)) {
+ return;
+ }
+ assert truncateBlock instanceof BlockInfo : "should be BlockInfo";
+ addDeleteBlock((BlockInfo) truncateBlock);
}
public void addUpdateReplicationFactor(BlockInfo block, short targetRepl) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/def87db9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index d4215e8..51a94e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -1155,6 +1156,46 @@ public class TestFileTruncate {
fs.delete(parent, true);
}
+ /**
+ * While rolling upgrade is in-progress the test truncates a file
+ * such that copy-on-truncate is triggered, then deletes the file,
+ * and makes sure that no blocks involved in truncate are hanging around.
+ */
+ @Test
+ public void testTruncateWithRollingUpgrade() throws Exception {
+ final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0));
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ //start rolling upgrade
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"});
+ assertEquals("could not prepare for rolling upgrade", 0, status);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ Path dir = new Path("/testTruncateWithRollingUpgrade");
+ fs.mkdirs(dir);
+ final Path p = new Path(dir, "file");
+ final byte[] data = new byte[3];
+ ThreadLocalRandom.current().nextBytes(data);
+ writeContents(data, data.length, p);
+
+ assertEquals("block num should 1", 1,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+
+ final boolean isReady = fs.truncate(p, 2);
+ assertFalse("should be copy-on-truncate", isReady);
+ assertEquals("block num should 2", 2,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ fs.delete(p, true);
+
+ assertEquals("block num should 0", 0,
+ cluster.getNamesystem().getFSDirectory().getBlockManager()
+ .getTotalBlocks());
+ status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"});
+ assertEquals("could not finalize rolling upgrade", 0, status);
+ }
+
static void writeContents(byte[] contents, int fileLength, Path p)
throws IOException {
FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org