You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/11/29 20:44:30 UTC
[14/50] [abbrv] hadoop git commit: HDFS-14095. EC: Track Erasure
Coding commands in DFS statistics. Contributed by Ayush Saxena.
HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5347368
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5347368
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5347368
Branch: refs/heads/HDDS-4
Commit: f534736867eed962899615ca1b7eb68bcf591d17
Parents: d0edd37
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Nov 30 00:18:27 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Nov 30 00:18:27 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/DFSOpsCountStatistics.java | 9 +++
.../hadoop/hdfs/DistributedFileSystem.java | 18 ++++++
.../hadoop/hdfs/TestDistributedFileSystem.java | 63 +++++++++++++++++++-
3 files changed, 89 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index 3dcf13b..b9852ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
/** This is for counting distributed file system operations. */
public enum OpType {
+ ADD_EC_POLICY("op_add_ec_policy"),
ALLOW_SNAPSHOT("op_allow_snapshot"),
APPEND(CommonStatisticNames.OP_APPEND),
CONCAT("op_concat"),
@@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics {
CREATE_SYM_LINK("op_create_symlink"),
DELETE(CommonStatisticNames.OP_DELETE),
DELETE_SNAPSHOT("op_delete_snapshot"),
+ DISABLE_EC_POLICY("op_disable_ec_policy"),
DISALLOW_SNAPSHOT("op_disallow_snapshot"),
+ ENABLE_EC_POLICY("op_enable_ec_policy"),
EXISTS(CommonStatisticNames.OP_EXISTS),
GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
+ GET_EC_CODECS("op_get_ec_codecs"),
+ GET_EC_POLICY("op_get_ec_policy"),
+ GET_EC_POLICIES("op_get_ec_policies"),
GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
GET_FILE_LINK_STATUS("op_get_file_link_status"),
@@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics {
REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
+ REMOVE_EC_POLICY("op_remove_ec_policy"),
REMOVE_XATTR("op_remove_xattr"),
RENAME(CommonStatisticNames.OP_RENAME),
RENAME_SNAPSHOT("op_rename_snapshot"),
RESOLVE_LINK("op_resolve_link"),
SET_ACL(CommonStatisticNames.OP_SET_ACL),
+ SET_EC_POLICY("op_set_ec_policy"),
SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
SET_REPLICATION("op_set_replication"),
@@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
GET_SNAPSHOT_DIFF("op_get_snapshot_diff"),
GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"),
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
+ UNSET_EC_POLICY("op_unset_ec_policy"),
UNSET_STORAGE_POLICY("op_unset_storage_policy");
private static final Map<String, OpType> SYMBOL_MAP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index ca1546c..7dd02bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2845,6 +2845,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void setErasureCodingPolicy(final Path path,
final String ecPolicyName) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
@@ -2912,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY);
Path absF = fixRelativePart(path);
return new FileSystemLinkResolver<ErasureCodingPolicy>() {
@Override
@@ -2943,6 +2947,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES);
return Arrays.asList(dfs.getErasureCodingPolicies());
}
@@ -2955,6 +2961,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Map<String, String> getAllErasureCodingCodecs()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS);
return dfs.getErasureCodingCodecs();
}
@@ -2971,6 +2979,8 @@ public class DistributedFileSystem extends FileSystem
*/
public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY);
return dfs.addErasureCodingPolicies(policies);
}
@@ -2982,6 +2992,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY);
dfs.removeErasureCodingPolicy(ecPolicyName);
}
@@ -2993,6 +3005,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void enableErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY);
dfs.enableErasureCodingPolicy(ecPolicyName);
}
@@ -3004,6 +3018,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void disableErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY);
dfs.disableErasureCodingPolicy(ecPolicyName);
}
@@ -3014,6 +3030,8 @@ public class DistributedFileSystem extends FileSystem
* @throws IOException
*/
public void unsetErasureCodingPolicy(final Path path) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index cae0fbf..97ced7a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -773,7 +773,68 @@ public class TestDistributedFileSystem {
} finally {
if (cluster != null) cluster.shutdown();
}
-
+ }
+
+ @Test
+ public void testECStatistics() throws IOException {
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(getTestConfiguration()).build()) {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ Path dir = new Path("/test");
+ dfs.mkdirs(dir);
+ int readOps = 0;
+ int writeOps = 0;
+ FileSystem.clearStatistics();
+
+ long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY);
+ dfs.enableErasureCodingPolicy("RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.SET_EC_POLICY);
+ dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_POLICY);
+ dfs.getErasureCodingPolicy(dir);
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.UNSET_EC_POLICY);
+ dfs.unsetErasureCodingPolicy(dir);
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_POLICIES);
+ dfs.getAllErasureCodingPolicies();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_CODECS);
+ dfs.getAllErasureCodingCodecs();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1);
+
+ ErasureCodingPolicy newPolicy =
+ new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024);
+
+ opCount = getOpStatistics(OpType.ADD_EC_POLICY);
+ dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy});
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.REMOVE_EC_POLICY);
+ dfs.removeErasureCodingPolicy("RS-5-3-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.DISABLE_EC_POLICY);
+ dfs.disableErasureCodingPolicy("RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1);
+ }
}
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org