You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2018/01/18 23:47:54 UTC
[23/49] hadoop git commit: HDFS-12972. RBF: Display mount table quota
info in Web UI and admin command. Contributed by Yiqun Lin.
HDFS-12972. RBF: Display mount table quota info in Web UI and admin command. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9afb8025
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9afb8025
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9afb8025
Branch: refs/heads/YARN-7402
Commit: 9afb8025d6549f0ade0ae7d36f5e67cd20c500f4
Parents: 2dcfc18
Author: Yiqun Lin <yq...@apache.org>
Authored: Sat Jan 13 13:16:43 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Sat Jan 13 13:16:43 2018 +0800
----------------------------------------------------------------------
.../federation/router/RouterQuotaUsage.java | 31 ++++++++++++++++++++
.../federation/store/records/MountTable.java | 10 +++++--
.../store/records/impl/pb/MountTablePBImpl.java | 21 ++++++++-----
.../hdfs/tools/federation/RouterAdmin.java | 16 ++++++----
.../main/webapps/router/federationhealth.html | 2 ++
.../metrics/TestFederationMetrics.java | 1 +
.../federation/router/TestRouterAdminCLI.java | 19 ++++++++++--
7 files changed, 83 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
index 55bfc48..eedd80f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
@@ -19,14 +19,20 @@ package org.apache.hadoop.hdfs.server.federation.router;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.util.StringUtils;
/**
* The subclass of {@link QuotaUsage} used in Router-based federation.
*/
public final class RouterQuotaUsage extends QuotaUsage {
+
+ /** Default quota usage count. */
+ public static final long QUOTA_USAGE_COUNT_DEFAULT = 0;
+
private RouterQuotaUsage(Builder builder) {
super(builder);
}
@@ -85,4 +91,29 @@ public final class RouterQuotaUsage extends QuotaUsage {
throw new DSQuotaExceededException(getSpaceQuota(), getSpaceConsumed());
}
}
+
+ @Override
+ public String toString() {
+ String nsQuota = String.valueOf(getQuota());
+ String nsCount = String.valueOf(getFileAndDirectoryCount());
+ if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ nsQuota = "-";
+ nsCount = "-";
+ }
+
+ String ssQuota = StringUtils.byteDesc(getSpaceQuota());
+ String ssCount = StringUtils.byteDesc(getSpaceConsumed());
+ if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ ssQuota = "-";
+ ssCount = "-";
+ }
+
+ StringBuilder str = new StringBuilder();
+ str.append("[NsQuota: ").append(nsQuota).append("/")
+ .append(nsCount);
+ str.append(", SsQuota: ").append(ssQuota)
+ .append("/").append(ssCount)
+ .append("]");
+ return str.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
index 53ad1e1..3a99f14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -144,8 +144,10 @@ public abstract class MountTable extends BaseRecord {
// Set quota for mount table
RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
- .fileAndDirectoryCount(0).quota(HdfsConstants.QUOTA_DONT_SET)
- .spaceConsumed(0).spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
+ .fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
+ .quota(HdfsConstants.QUOTA_DONT_SET)
+ .spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
+ .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
record.setQuota(quota);
// Validate
@@ -325,6 +327,10 @@ public abstract class MountTable extends BaseRecord {
sb.append("[mode:").append(this.getMode()).append("]");
}
+ if (this.getQuota() != null) {
+ sb.append("[quota:").append(this.getQuota()).append("]");
+ }
+
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
index d9e9555..3beeca3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
@@ -255,16 +256,22 @@ public class MountTablePBImpl extends MountTable implements PBRecord {
@Override
public RouterQuotaUsage getQuota() {
MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasQuota()) {
- return null;
+
+ long nsQuota = HdfsConstants.QUOTA_DONT_SET;
+ long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+ long ssQuota = HdfsConstants.QUOTA_DONT_SET;
+ long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+ if (proto.hasQuota()) {
+ QuotaUsageProto quotaProto = proto.getQuota();
+ nsQuota = quotaProto.getQuota();
+ nsCount = quotaProto.getFileAndDirectoryCount();
+ ssQuota = quotaProto.getSpaceQuota();
+ ssCount = quotaProto.getSpaceConsumed();
}
- QuotaUsageProto quotaProto = proto.getQuota();
RouterQuotaUsage.Builder builder = new RouterQuotaUsage.Builder()
- .fileAndDirectoryCount(quotaProto.getFileAndDirectoryCount())
- .quota(quotaProto.getQuota())
- .spaceConsumed(quotaProto.getSpaceConsumed())
- .spaceQuota(quotaProto.getSpaceQuota());
+ .quota(nsQuota).fileAndDirectoryCount(nsCount).spaceQuota(ssQuota)
+ .spaceConsumed(ssCount);
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 0681ed5..d5a2d77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
@@ -395,8 +396,8 @@ public class RouterAdmin extends Configured implements Tool {
private static void printMounts(List<MountTable> entries) {
System.out.println("Mount Table Entries:");
System.out.println(String.format(
- "%-25s %-25s %-25s %-25s %-25s",
- "Source", "Destinations", "Owner", "Group", "Mode"));
+ "%-25s %-25s %-25s %-25s %-25s %-25s",
+ "Source", "Destinations", "Owner", "Group", "Mode", "Quota/Usage"));
for (MountTable entry : entries) {
StringBuilder destBuilder = new StringBuilder();
for (RemoteLocation location : entry.getDestinations()) {
@@ -409,8 +410,10 @@ public class RouterAdmin extends Configured implements Tool {
System.out.print(String.format("%-25s %-25s", entry.getSourcePath(),
destBuilder.toString()));
- System.out.println(String.format(" %-25s %-25s %-25s",
+ System.out.print(String.format(" %-25s %-25s %-25s",
entry.getOwnerName(), entry.getGroupName(), entry.getMode()));
+
+ System.out.println(String.format(" %-25s", entry.getQuota()));
}
}
@@ -436,7 +439,8 @@ public class RouterAdmin extends Configured implements Tool {
} else if (parameters[i].equals("-ssQuota")) {
i++;
try {
- ssQuota = Long.parseLong(parameters[i]);
+ ssQuota = StringUtils.TraditionalBinaryPrefix
+ .string2long(parameters[i]);
} catch (Exception e) {
System.err.println("Cannot parse ssQuota: " + parameters[i]);
}
@@ -499,11 +503,11 @@ public class RouterAdmin extends Configured implements Tool {
// If nsQuota or ssQuota was unset, reset corresponding usage
// value to zero.
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
- nsCount = 0;
+ nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
}
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
- ssCount = 0;
+ ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
}
RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
index 9c28975..ac5bb49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
@@ -338,6 +338,7 @@
<th>Owner</th>
<th>Group</th>
<th>Permission</th>
+ <th>Quota/Usage</th>
<th>Date Modified</th>
<th>Date Created</th>
</tr>
@@ -353,6 +354,7 @@
<td>{ownerName}</td>
<td>{groupName}</td>
<td>{mode}</td>
+ <td>{quota}</td>
<td>{dateModified}</td>
<td>{dateCreated}</td>
</tr>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
index b445df5..06d1890 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
@@ -87,6 +87,7 @@ public class TestFederationMetrics extends TestMetricsBase {
assertEquals(entry.getOwnerName(), json.getString("ownerName"));
assertEquals(entry.getGroupName(), json.getString("groupName"));
assertEquals(entry.getMode().toString(), json.getString("mode"));
+ assertEquals(entry.getQuota().toString(), json.getString("quota"));
assertNotNullAndNotEmpty(json.getString("dateCreated"));
assertNotNullAndNotEmpty(json.getString("dateModified"));
match++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9afb8025/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index ec47a41..265fcde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -313,9 +313,11 @@ public class TestRouterAdminCLI {
RouterQuotaUsage quotaUsage = mountTable.getQuota();
// verify the default quota set
- assertEquals(0, quotaUsage.getFileAndDirectoryCount());
+ assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
+ quotaUsage.getFileAndDirectoryCount());
assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
- assertEquals(0, quotaUsage.getSpaceConsumed());
+ assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
+ quotaUsage.getSpaceConsumed());
assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
long nsQuota = 50;
@@ -334,6 +336,19 @@ public class TestRouterAdminCLI {
assertEquals(nsQuota, quotaUsage.getQuota());
assertEquals(ssQuota, quotaUsage.getSpaceQuota());
+ // use quota string for setting ss quota
+ String newSsQuota = "2m";
+ argv = new String[] {"-setQuota", src, "-ssQuota", newSsQuota};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ mountTable = getResponse.getEntries().get(0);
+ quotaUsage = mountTable.getQuota();
+ // verify if ss quota is correctly set
+ assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota());
+
// test clrQuota command
argv = new String[] {"-clrQuota", src};
assertEquals(0, ToolRunner.run(admin, argv));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org