You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2019/09/05 23:54:36 UTC
[hadoop] branch branch-2.8 updated (12ec7c3 -> f94521b)
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a change to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 12ec7c3 HDFS-14633. The StorageType quota and consume in QuotaFeature is not handled for rename. Contributed by Jinglun.
new 891a713 Revert "HDFS-14633. The StorageType quota and consume in QuotaFeature is not handled for rename. Contributed by Jinglun."
new f94521b HDFS-14633. The StorageType quota and consume in QuotaFeature is not handled for rename. Contributed by Jinglun.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/test/java/org/apache/hadoop/hdfs/TestQuota.java | 13 -------------
1 file changed, 13 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: Revert "HDFS-14633. The StorageType quota and
consume in QuotaFeature is not handled for rename. Contributed by Jinglun."
Posted by xy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 891a7135b50d41e8ae23f8df36eae52608798337
Author: Xiaoyu Yao <xy...@apache.org>
AuthorDate: Thu Sep 5 16:38:42 2019 -0700
Revert "HDFS-14633. The StorageType quota and consume in QuotaFeature is not handled for rename. Contributed by Jinglun."
This reverts commit 12ec7c301ff19b004fd94e3f0af7238f8050bccc.
---
.../hadoop/hdfs/server/namenode/FSDirRenameOp.java | 7 +--
.../hadoop/hdfs/server/namenode/FSDirectory.java | 5 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 63 +---------------------
3 files changed, 3 insertions(+), 72 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index 23e0d16..e6f1dd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -78,12 +78,7 @@ class FSDirRenameOp {
while(src.getINode(i) == dst.getINode(i)) { i++; }
// src[i - 1] is the last common ancestor.
BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
- // Assume dstParent existence check done by callers.
- INode dstParent = dst.getINode(-2);
- // Use the destination parent's storage policy for quota delta verify.
- final QuotaCounts delta = src.getLastINode()
- .computeQuotaUsage(bsps, dstParent.getStoragePolicyID(), false,
- Snapshot.CURRENT_STATE_ID);
+ final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);
// Reduce the required quota by dst that is being removed
final INode dstINode = dst.getLastINode();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 31b9cce..437d359 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo.UpdatedReplicationInfo;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ByteArray;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -1107,9 +1106,7 @@ public class FSDirectory implements Closeable {
// always verify inode name
verifyINodeName(inode.getLocalNameBytes());
- final QuotaCounts counts = inode
- .computeQuotaUsage(getBlockStoragePolicySuite(),
- parent.getStoragePolicyID(), false, Snapshot.CURRENT_STATE_ID);
+ final QuotaCounts counts = inode.computeQuotaUsage(getBlockStoragePolicySuite());
updateCount(existing, pos, counts, checkQuota);
boolean isRename = (inode.getParent() != null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index cdd787a..c46620d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -1538,57 +1539,6 @@ public class TestQuota {
}
@Test
- public void testClrQuotaOnRoot() throws Exception {
- long orignalQuota = dfs.getQuotaUsage(new Path("/")).getQuota();
- DFSAdmin admin = new DFSAdmin(conf);
- String[] args;
- args = new String[] {"-setQuota", "3K", "/"};
- runCommand(admin, args, false);
- assertEquals(3 * 1024, dfs.getQuotaUsage(new Path("/")).getQuota());
- args = new String[] {"-clrQuota", "/"};
- runCommand(admin, args, false);
- assertEquals(orignalQuota, dfs.getQuotaUsage(new Path("/")).getQuota());
- }
-
- @Test
- public void testRename() throws Exception {
- int fileLen = 1024;
- short replication = 3;
-
- final Path parent = new Path(PathUtils.getTestDir(getClass()).getPath(),
- GenericTestUtils.getMethodName());
- assertTrue(dfs.mkdirs(parent));
-
- final Path srcDir = new Path(parent, "src-dir");
- Path file = new Path(srcDir, "file1");
- DFSTestUtil.createFile(dfs, file, fileLen, replication, 0);
- dfs.setStoragePolicy(srcDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
-
- final Path dstDir = new Path(parent, "dst-dir");
- assertTrue(dfs.mkdirs(dstDir));
- dfs.setStoragePolicy(dstDir, HdfsConstants.ALLSSD_STORAGE_POLICY_NAME);
-
- dfs.setQuota(srcDir, 100000, 100000);
- dfs.setQuota(dstDir, 100000, 100000);
-
- Path dstFile = new Path(dstDir, "file1");
- // Test quota check of rename. Expect a QuotaExceedException.
- dfs.setQuotaByStorageType(dstDir, StorageType.SSD, 10);
- try {
- dfs.rename(file, dstFile);
- fail("Expect QuotaExceedException.");
- } catch (QuotaExceededException qe) {
- }
-
- // Set enough quota, expect a successful rename.
- dfs.setQuotaByStorageType(dstDir, StorageType.SSD, fileLen * replication);
- dfs.rename(file, dstFile);
- // Verify the storage type usage is properly updated on source and dst.
- checkQuotaAndCount(dfs, srcDir);
- checkQuotaAndCount(dfs, dstDir);
- }
-
- @Test
public void testSpaceQuotaExceptionOnAppend() throws Exception {
GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.TRACE);
GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.TRACE);
@@ -1656,15 +1606,4 @@ public class TestQuota {
}
scanner.close();
}
-
- // quota and count should match.
- private void checkQuotaAndCount(DistributedFileSystem fs, Path path)
- throws IOException {
- QuotaUsage qu = fs.getQuotaUsage(path);
- ContentSummary cs = fs.getContentSummary(path);
- for (StorageType st : StorageType.values()) {
- // it will fail here, because the quota and consume is not handled right.
- assertEquals(qu.getTypeConsumed(st), cs.getTypeConsumed(st));
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-14633. The StorageType quota and consume in
QuotaFeature is not handled for rename. Contributed by Jinglun.
Posted by xy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f94521b06581d42e2546e5e7ee09707b0a6f5291
Author: Xiaoyu Yao <xy...@apache.org>
AuthorDate: Fri Aug 30 16:46:04 2019 -0700
HDFS-14633. The StorageType quota and consume in QuotaFeature is not handled for rename. Contributed by Jinglun.
(cherry picked from commit 62d71fbac3789c7d484bc76ced9ec7fa6ff94de1)
---
.../hadoop/hdfs/server/namenode/FSDirRenameOp.java | 7 ++-
.../hadoop/hdfs/server/namenode/FSDirectory.java | 5 ++-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 50 +++++++++++++++++++++-
3 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index e6f1dd1..23e0d16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -78,7 +78,12 @@ class FSDirRenameOp {
while(src.getINode(i) == dst.getINode(i)) { i++; }
// src[i - 1] is the last common ancestor.
BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
- final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);
+ // Assume dstParent existence check done by callers.
+ INode dstParent = dst.getINode(-2);
+ // Use the destination parent's storage policy for quota delta verify.
+ final QuotaCounts delta = src.getLastINode()
+ .computeQuotaUsage(bsps, dstParent.getStoragePolicyID(), false,
+ Snapshot.CURRENT_STATE_ID);
// Reduce the required quota by dst that is being removed
final INode dstINode = dst.getLastINode();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 437d359..31b9cce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo.UpdatedReplicationInfo;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ByteArray;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -1106,7 +1107,9 @@ public class FSDirectory implements Closeable {
// always verify inode name
verifyINodeName(inode.getLocalNameBytes());
- final QuotaCounts counts = inode.computeQuotaUsage(getBlockStoragePolicySuite());
+ final QuotaCounts counts = inode
+ .computeQuotaUsage(getBlockStoragePolicySuite(),
+ parent.getStoragePolicyID(), false, Snapshot.CURRENT_STATE_ID);
updateCount(existing, pos, counts, checkQuota);
boolean isRename = (inode.getParent() != null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index c46620d..079ce1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -1539,6 +1538,44 @@ public class TestQuota {
}
@Test
+ public void testRename() throws Exception {
+ int fileLen = 1024;
+ short replication = 3;
+
+ final Path parent = new Path(PathUtils.getTestDir(getClass()).getPath(),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(parent));
+
+ final Path srcDir = new Path(parent, "src-dir");
+ Path file = new Path(srcDir, "file1");
+ DFSTestUtil.createFile(dfs, file, fileLen, replication, 0);
+ dfs.setStoragePolicy(srcDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
+
+ final Path dstDir = new Path(parent, "dst-dir");
+ assertTrue(dfs.mkdirs(dstDir));
+ dfs.setStoragePolicy(dstDir, HdfsConstants.ALLSSD_STORAGE_POLICY_NAME);
+
+ dfs.setQuota(srcDir, 100000, 100000);
+ dfs.setQuota(dstDir, 100000, 100000);
+
+ Path dstFile = new Path(dstDir, "file1");
+ // Test quota check of rename. Expect a QuotaExceedException.
+ dfs.setQuotaByStorageType(dstDir, StorageType.SSD, 10);
+ try {
+ dfs.rename(file, dstFile);
+ fail("Expect QuotaExceedException.");
+ } catch (QuotaExceededException qe) {
+ }
+
+ // Set enough quota, expect a successful rename.
+ dfs.setQuotaByStorageType(dstDir, StorageType.SSD, fileLen * replication);
+ dfs.rename(file, dstFile);
+ // Verify the storage type usage is properly updated on source and dst.
+ checkQuotaAndCount(dfs, srcDir);
+ checkQuotaAndCount(dfs, dstDir);
+ }
+
+ @Test
public void testSpaceQuotaExceptionOnAppend() throws Exception {
GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.TRACE);
GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.TRACE);
@@ -1606,4 +1643,15 @@ public class TestQuota {
}
scanner.close();
}
+
+ // quota and count should match.
+ private void checkQuotaAndCount(DistributedFileSystem fs, Path path)
+ throws IOException {
+ QuotaUsage qu = fs.getQuotaUsage(path);
+ ContentSummary cs = fs.getContentSummary(path);
+ for (StorageType st : StorageType.values()) {
+ // it will fail here, because the quota and consume is not handled right.
+ assertEquals(qu.getTypeConsumed(st), cs.getTypeConsumed(st));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org