You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2020/01/09 06:58:13 UTC
[hadoop] branch branch-3.1 updated: HDFS-14957. INodeReference
Space Consumed was not same in QuotaUsage and ContentSummary. Contributed
by hemanthboyina.
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 09d3707 HDFS-14957. INodeReference Space Consumed was not same in QuotaUsage and ContentSummary. Contributed by hemanthboyina.
09d3707 is described below
commit 09d37078757df0d3ddc97d70cab7b8d8651d814c
Author: Surendra Singh Lilhore <su...@apache.org>
AuthorDate: Thu Jan 9 12:04:05 2020 +0530
HDFS-14957. INodeReference Space Consumed was not same in QuotaUsage and ContentSummary. Contributed by hemanthboyina.
---
.../hadoop/hdfs/server/namenode/INodeFile.java | 42 ++++++++++++++++++++--
.../namenode/snapshot/TestRenameWithSnapshots.java | 26 +++++++++++++-
2 files changed, 65 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index ce654b7..67c86b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -873,8 +873,18 @@ public class INodeFile extends INodeWithAdditionalFields
counts.addContent(Content.FILE, 1);
final long fileLen = computeFileSize(snapshotId);
counts.addContent(Content.LENGTH, fileLen);
- counts.addContent(Content.DISKSPACE, storagespaceConsumed(null)
- .getStorageSpace());
+
+ FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
+ if (sf == null) {
+ counts.addContent(Content.DISKSPACE,
+ storagespaceConsumed(null).getStorageSpace());
+ } else if (isStriped()) {
+ counts.addContent(Content.DISKSPACE,
+ storagespaceConsumedStriped().getStorageSpace());
+ } else {
+ long diskSpaceQuota = getDiskSpaceQuota(counts, sf, snapshotId);
+ counts.addContent(Content.DISKSPACE, diskSpaceQuota);
+ }
if (getStoragePolicyID() != BLOCK_STORAGE_POLICY_ID_UNSPECIFIED){
BlockStoragePolicy bsp = summary.getBlockStoragePolicySuite().
@@ -890,6 +900,34 @@ public class INodeFile extends INodeWithAdditionalFields
return summary;
}
+ /**
+ * Compute disk space consumed by all the blocks in snapshots.
+ */
+ private long getDiskSpaceQuota(ContentCounts counts,
+ FileWithSnapshotFeature sf, int lastSnapshotId) {
+ FileDiffList fileDiffList = sf.getDiffs();
+ int last = fileDiffList.getLastSnapshotId();
+
+ if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
+ || last == Snapshot.CURRENT_STATE_ID) {
+ return storagespaceConsumed(null).getStorageSpace();
+ }
+
+ final long ssDeltaNoReplication;
+ short replication;
+
+ if (last < lastSnapshotId) {
+ ssDeltaNoReplication = computeFileSize(true, false);
+ replication = getFileReplication();
+ } else {
+ int sid = fileDiffList.getSnapshotById(lastSnapshotId);
+ ssDeltaNoReplication = computeFileSize(sid);
+ replication = getFileReplication(sid);
+ }
+
+ return ssDeltaNoReplication * replication;
+ }
+
/** The same as computeFileSize(null). */
public final long computeFileSize() {
return computeFileSize(CURRENT_STATE_ID);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index cb4b3c3..589bbb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -2473,4 +2473,28 @@ public class TestRenameWithSnapshots {
output.println(b);
return b;
}
-}
+
+ /**
+ * Test getContentsummary and getQuotausage for an INodeReference.
+ */
+ @Test(timeout = 300000)
+ public void testQuotaForRenameFileInSnapshot() throws Exception {
+ final Path snapshotDir = new Path("/testRenameWithSnapshot");
+ hdfs.mkdirs(snapshotDir, new FsPermission((short) 0777));
+ final Path file = new Path(snapshotDir, "file");
+ DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPL, SEED);
+ hdfs.allowSnapshot(snapshotDir);
+ hdfs.createSnapshot(snapshotDir, "s0");
+ hdfs.mkdirs(new Path("/dir1"));
+
+ // Truncate a file which exists in snapshot , that is an
+ // INodeReference
+ hdfs.truncate(file, 10);
+ hdfs.rename(file, new Path("/dir1"));
+ assertEquals(hdfs.getContentSummary(new Path("/")).getSpaceConsumed(),
+ hdfs.getQuotaUsage(new Path("/")).getSpaceConsumed());
+ assertEquals(
+ hdfs.getContentSummary(new Path("/")).getFileAndDirectoryCount(),
+ hdfs.getQuotaUsage(new Path("/")).getFileAndDirectoryCount());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org