You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/11/18 11:17:44 UTC
[hadoop] branch trunk updated: HDFS-14955. RBF: getQuotaUsage() on
mount point should return global quota. Contributed by Jinglun.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 12617fa HDFS-14955. RBF: getQuotaUsage() on mount point should return global quota. Contributed by Jinglun.
12617fa is described below
commit 12617fad2eb32108412dac9ecee286de6641d060
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Mon Nov 18 16:04:52 2019 +0530
HDFS-14955. RBF: getQuotaUsage() on mount point should return global quota. Contributed by Jinglun.
---
.../hdfs/server/federation/router/Quota.java | 17 +++++++--
.../router/RouterQuotaUpdateService.java | 2 +-
.../server/federation/router/TestRouterQuota.java | 42 ++++++++++++++++++++++
3 files changed, 57 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
index ed69726..ee65608 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
@@ -125,7 +125,7 @@ public class Quota {
* @throws IOException If the quota system is disabled.
*/
public QuotaUsage getQuotaUsage(String path) throws IOException {
- return aggregateQuota(getEachQuotaUsage(path));
+ return aggregateQuota(path, getEachQuotaUsage(path));
}
/**
@@ -234,20 +234,26 @@ public class Quota {
/**
* Aggregate quota that queried from sub-clusters.
+ * @param path Federation path of the results.
* @param results Quota query result.
* @return Aggregated Quota.
*/
- QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
+ QuotaUsage aggregateQuota(String path,
+ Map<RemoteLocation, QuotaUsage> results) throws IOException {
long nsCount = 0;
long ssCount = 0;
long nsQuota = HdfsConstants.QUOTA_RESET;
long ssQuota = HdfsConstants.QUOTA_RESET;
boolean hasQuotaUnset = false;
+ boolean isMountEntry = isMountEntry(path);
for (Map.Entry<RemoteLocation, QuotaUsage> entry : results.entrySet()) {
RemoteLocation loc = entry.getKey();
QuotaUsage usage = entry.getValue();
- if (usage != null) {
+ if (isMountEntry) {
+ nsCount += usage.getFileAndDirectoryCount();
+ ssCount += usage.getSpaceConsumed();
+ } else if (usage != null) {
// If quota is not set in real FileSystem, the usage
// value will return -1.
if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) {
@@ -266,6 +272,11 @@ public class Quota {
}
}
+ if (isMountEntry) {
+ QuotaUsage quota = getGlobalQuota(path);
+ nsQuota = quota.getQuota();
+ ssQuota = quota.getSpaceQuota();
+ }
QuotaUsage.Builder builder = new QuotaUsage.Builder()
.fileAndDirectoryCount(nsCount).spaceConsumed(ssCount);
if (hasQuotaUnset) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
index 7982bc9..e5d4472 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
@@ -110,7 +110,7 @@ public class RouterQuotaUpdateService extends PeriodicService {
Quota quotaModule = this.rpcServer.getQuotaModule();
Map<RemoteLocation, QuotaUsage> usageMap =
quotaModule.getEachQuotaUsage(src);
- currentQuotaUsage = quotaModule.aggregateQuota(usageMap);
+ currentQuotaUsage = quotaModule.aggregateQuota(src, usageMap);
remoteQuotaUsage.putAll(usageMap);
} catch (IOException ioe) {
LOG.error("Unable to get quota usage for " + src, ioe);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
index 0b9185a..19f7ad6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
@@ -964,6 +964,48 @@ public class TestRouterQuota {
assertEquals(-1, qu.getSpaceQuota());
}
+ @Test
+ public void testGetQuotaUsageOnMountPoint() throws Exception {
+ long nsQuota = 5;
+ long ssQuota = 3 * BLOCK_SIZE;
+ prepareGlobalQuotaTestMountTable(nsQuota, ssQuota);
+
+ final FileSystem routerFs = routerContext.getFileSystem();
+ // Verify getQuotaUsage() of the mount point /dir-1.
+ QuotaUsage quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1"));
+ assertEquals(nsQuota, quotaUsage.getQuota());
+ assertEquals(ssQuota, quotaUsage.getSpaceQuota());
+ // Verify getQuotaUsage() of the mount point /dir-1/dir-2.
+ quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-2"));
+ assertEquals(nsQuota, quotaUsage.getQuota());
+ assertEquals(ssQuota * 2, quotaUsage.getSpaceQuota());
+ // Verify getQuotaUsage() of the mount point /dir-1/dir-2/dir-3
+ quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-2/dir-3"));
+ assertEquals(nsQuota, quotaUsage.getQuota());
+ assertEquals(ssQuota * 2, quotaUsage.getSpaceQuota());
+ // Verify getQuotaUsage() of the mount point /dir-4
+ quotaUsage = routerFs.getQuotaUsage(new Path("/dir-4"));
+ assertEquals(-1, quotaUsage.getQuota());
+ assertEquals(-1, quotaUsage.getSpaceQuota());
+
+ routerFs.mkdirs(new Path("/dir-1/dir-normal"));
+ try {
+ // Verify getQuotaUsage() of the normal path /dir-1/dir-normal.
+ quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-normal"));
+ assertEquals(-1, quotaUsage.getQuota());
+ assertEquals(-1, quotaUsage.getSpaceQuota());
+
+ routerFs.setQuota(new Path("/dir-1/dir-normal"), 100, 200);
+ // Verify getQuotaUsage() of the normal path /dir-1/dir-normal.
+ quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-normal"));
+ assertEquals(100, quotaUsage.getQuota());
+ assertEquals(200, quotaUsage.getSpaceQuota());
+ } finally {
+ // clear normal path.
+ routerFs.delete(new Path("/dir-1/dir-normal"), true);
+ }
+ }
+
/**
* Add three mount tables.
* /dir-1 --> ns0---/dir-1 [nsQuota, ssQuota]
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org