You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/04/02 09:21:14 UTC
[hadoop] 04/09: HDFS-15510. RBF: Quota and Content Summary was not
correct in Multiple Destinations. Contributed by Hemanth Boyina.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f5cc1540b4510b6da72b46d075b4463e352d4848
Author: Takanobu Asanuma <ta...@apache.org>
AuthorDate: Thu Aug 27 12:10:39 2020 +0900
HDFS-15510. RBF: Quota and Content Summary was not correct in Multiple Destinations. Contributed by Hemanth Boyina.
(cherry picked from commit ca8e7a77256003e11ab7e3d079ee4cf9f50080dd)
---
.../federation/router/RouterClientProtocol.java | 6 ++-
...erRPCMultipleDestinationMountTableResolver.java | 57 ++++++++++++++++++++++
2 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
index ec61258..2a0fe7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
@@ -1857,6 +1857,8 @@ public class RouterClientProtocol implements ClientProtocol {
/**
* Aggregate content summaries for each subcluster.
+ * If the mount point has multiple destinations
+ * add the quota set value only once.
*
* @param summaries Collection of individual summaries.
* @return Aggregated content summary.
@@ -1879,9 +1881,9 @@ public class RouterClientProtocol implements ClientProtocol {
length += summary.getLength();
fileCount += summary.getFileCount();
directoryCount += summary.getDirectoryCount();
- quota += summary.getQuota();
+ quota = summary.getQuota();
spaceConsumed += summary.getSpaceConsumed();
- spaceQuota += summary.getSpaceQuota();
+ spaceQuota = summary.getSpaceQuota();
// We return from the first response as we assume that the EC policy
// of each sub-cluster is same.
if (ecPolicy.isEmpty()) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
index bcab7bb..e644dec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
@@ -550,6 +550,63 @@ public class TestRouterRPCMultipleDestinationMountTableResolver {
assertEquals(-1, cs1.getSpaceQuota());
}
+ @Test
+ public void testContentSummaryWithMultipleDest() throws Exception {
+ MountTable addEntry;
+ long nsQuota = 5;
+ long ssQuota = 100;
+ Path path = new Path("/testContentSummaryWithMultipleDest");
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put("ns0", "/testContentSummaryWithMultipleDest");
+ destMap.put("ns1", "/testContentSummaryWithMultipleDest");
+ nnFs0.mkdirs(path);
+ nnFs1.mkdirs(path);
+ addEntry =
+ MountTable.newInstance("/testContentSummaryWithMultipleDest", destMap);
+ addEntry.setQuota(
+ new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota)
+ .build());
+ assertTrue(addMountTable(addEntry));
+ RouterQuotaUpdateService updateService =
+ routerContext.getRouter().getQuotaCacheUpdateService();
+ updateService.periodicInvoke();
+ ContentSummary cs = routerFs.getContentSummary(path);
+ assertEquals(nsQuota, cs.getQuota());
+ assertEquals(ssQuota, cs.getSpaceQuota());
+ ContentSummary ns0Cs = nnFs0.getContentSummary(path);
+ assertEquals(nsQuota, ns0Cs.getQuota());
+ assertEquals(ssQuota, ns0Cs.getSpaceQuota());
+ ContentSummary ns1Cs = nnFs1.getContentSummary(path);
+ assertEquals(nsQuota, ns1Cs.getQuota());
+ assertEquals(ssQuota, ns1Cs.getSpaceQuota());
+ }
+
+ @Test
+ public void testContentSummaryMultipleDestWithMaxValue()
+ throws Exception {
+ MountTable addEntry;
+ long nsQuota = Long.MAX_VALUE - 2;
+ long ssQuota = Long.MAX_VALUE - 2;
+ Path path = new Path("/testContentSummaryMultipleDestWithMaxValue");
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put("ns0", "/testContentSummaryMultipleDestWithMaxValue");
+ destMap.put("ns1", "/testContentSummaryMultipleDestWithMaxValue");
+ nnFs0.mkdirs(path);
+ nnFs1.mkdirs(path);
+ addEntry = MountTable
+ .newInstance("/testContentSummaryMultipleDestWithMaxValue", destMap);
+ addEntry.setQuota(
+ new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota)
+ .build());
+ assertTrue(addMountTable(addEntry));
+ RouterQuotaUpdateService updateService =
+ routerContext.getRouter().getQuotaCacheUpdateService();
+ updateService.periodicInvoke();
+ ContentSummary cs = routerFs.getContentSummary(path);
+ assertEquals(nsQuota, cs.getQuota());
+ assertEquals(ssQuota, cs.getSpaceQuota());
+ }
+
/**
* Test to verify rename operation on directories in case of multiple
* destinations.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org