You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/07/01 22:35:19 UTC
hadoop git commit: HADOOP-13305. Define common statistics names
across schemes. Contributed by Mingliang Liu.
Repository: hadoop
Updated Branches:
refs/heads/trunk c35a5a7a8 -> aa42c7a6d
HADOOP-13305. Define common statistics names across schemes. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa42c7a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa42c7a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa42c7a6
Branch: refs/heads/trunk
Commit: aa42c7a6dda23f9dd686cc844b31a5aeebe7e088
Parents: c35a5a7
Author: Jitendra Pandey <ji...@apache.org>
Authored: Fri Jul 1 15:34:03 2016 -0700
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Fri Jul 1 15:34:03 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/FileSystemStorageStatistics.java | 5 +
.../org/apache/hadoop/fs/StorageStatistics.java | 53 ++++++++++
.../hadoop/hdfs/DFSOpsCountStatistics.java | 102 ++++++++++---------
.../hadoop/hdfs/TestDFSOpsCountStatistics.java | 15 +++
.../hadoop/fs/s3a/S3AStorageStatistics.java | 5 +
.../org/apache/hadoop/fs/s3a/Statistic.java | 24 +++--
6 files changed, 145 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
index 7c08863..8a1eb54 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
@@ -119,6 +119,11 @@ public class FileSystemStorageStatistics extends StorageStatistics {
}
@Override
+ public String getScheme() {
+ return stats.getScheme();
+ }
+
+ @Override
public Iterator<LongStatistic> getLongStatistics() {
return new LongStatisticIterator(stats.getData());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
index 4bdef80..0971f10 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
@@ -27,6 +27,51 @@ import java.util.Iterator;
*/
@InterfaceAudience.Public
public abstract class StorageStatistics {
+
+ /**
+ * These are common statistic names.
+ *
+ * The following names are considered general and preserved across different
+ * StorageStatistics classes. When implementing a new StorageStatistics, it is
+ * highly recommended to use the common statistic names.
+ *
+ * When adding new common statistic name constants, please make them unique.
+ * By convention, they are implicitly unique:
+ * - the name of the constants are uppercase, words separated by underscores.
+ * - the value of the constants are lowercase of the constant names.
+ */
+ public interface CommonStatisticNames {
+ // The following names are for file system operation invocations
+ String OP_APPEND = "op_append";
+ String OP_COPY_FROM_LOCAL_FILE = "op_copy_from_local_file";
+ String OP_CREATE = "op_create";
+ String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive";
+ String OP_DELETE = "op_delete";
+ String OP_EXISTS = "op_exists";
+ String OP_GET_CONTENT_SUMMARY = "op_get_content_summary";
+ String OP_GET_FILE_CHECKSUM = "op_get_file_checksum";
+ String OP_GET_FILE_STATUS = "op_get_file_status";
+ String OP_GET_STATUS = "op_get_status";
+ String OP_GLOB_STATUS = "op_glob_status";
+ String OP_IS_FILE = "op_is_file";
+ String OP_IS_DIRECTORY = "op_is_directory";
+ String OP_LIST_FILES = "op_list_files";
+ String OP_LIST_LOCATED_STATUS = "op_list_located_status";
+ String OP_LIST_STATUS = "op_list_status";
+ String OP_MKDIRS = "op_mkdirs";
+ String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
+ String OP_OPEN = "op_open";
+ String OP_REMOVE_ACL = "op_remove_acl";
+ String OP_REMOVE_ACL_ENTRIES = "op_remove_acl_entries";
+ String OP_REMOVE_DEFAULT_ACL = "op_remove_default_acl";
+ String OP_RENAME = "op_rename";
+ String OP_SET_ACL = "op_set_acl";
+ String OP_SET_OWNER = "op_set_owner";
+ String OP_SET_PERMISSION = "op_set_permission";
+ String OP_SET_TIMES = "op_set_times";
+ String OP_TRUNCATE = "op_truncate";
+ }
+
/**
* A 64-bit storage statistic.
*/
@@ -68,6 +113,14 @@ public abstract class StorageStatistics {
}
/**
+ * @return the associated file system scheme if this is scheme specific,
+ * else return null.
+ */
+ public String getScheme() {
+ return null;
+ }
+
+ /**
* Get an iterator over all the currently tracked long statistics.
*
* The values returned will depend on the type of FileSystem or FileContext
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index a047d34..84fc925 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import java.util.EnumMap;
import java.util.Iterator;
@@ -37,55 +38,55 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class DFSOpsCountStatistics extends StorageStatistics {
- /** This is for counting file system operations. */
+ /** This is for counting distributed file system operations. */
public enum OpType {
- ALLOW_SNAPSHOT("allowSnapshot"),
- APPEND("append"),
- CONCAT("concat"),
- COPY_FROM_LOCAL_FILE("copyFromLocalFile"),
- CREATE("create"),
- CREATE_NON_RECURSIVE("createNonRecursive"),
- CREATE_SNAPSHOT("createSnapshot"),
- CREATE_SYM_LINK("createSymlink"),
- DELETE("delete"),
- DELETE_SNAPSHOT("deleteSnapshot"),
- DISALLOW_SNAPSHOT("disallowSnapshot"),
- EXISTS("exists"),
- GET_BYTES_WITH_FUTURE_GS("getBytesWithFutureGenerationStamps"),
- GET_CONTENT_SUMMARY("getContentSummary"),
- GET_FILE_BLOCK_LOCATIONS("getFileBlockLocations"),
- GET_FILE_CHECKSUM("getFileChecksum"),
- GET_FILE_LINK_STATUS("getFileLinkStatus"),
- GET_FILE_STATUS("getFileStatus"),
- GET_LINK_TARGET("getLinkTarget"),
- GET_QUOTA_USAGE("getQuotaUsage"),
- GET_STATUS("getStatus"),
- GET_STORAGE_POLICIES("getStoragePolicies"),
- GET_STORAGE_POLICY("getStoragePolicy"),
- GET_XATTR("getXAttr"),
- LIST_LOCATED_STATUS("listLocatedStatus"),
- LIST_STATUS("listStatus"),
- MKDIRS("mkdirs"),
- MODIFY_ACL_ENTRIES("modifyAclEntries"),
- OPEN("open"),
- PRIMITIVE_CREATE("primitiveCreate"),
- PRIMITIVE_MKDIR("primitiveMkdir"),
- REMOVE_ACL("removeAcl"),
- REMOVE_ACL_ENTRIES("removeAclEntries"),
- REMOVE_DEFAULT_ACL("removeDefaultAcl"),
- REMOVE_XATTR("removeXAttr"),
- RENAME("rename"),
- RENAME_SNAPSHOT("renameSnapshot"),
- RESOLVE_LINK("resolveLink"),
- SET_ACL("setAcl"),
- SET_OWNER("setOwner"),
- SET_PERMISSION("setPermission"),
- SET_REPLICATION("setReplication"),
- SET_STORAGE_POLICY("setStoragePolicy"),
- SET_TIMES("setTimes"),
- SET_XATTR("setXAttr"),
- TRUNCATE("truncate"),
- UNSET_STORAGE_POLICY("unsetStoragePolicy");
+ ALLOW_SNAPSHOT("op_allow_snapshot"),
+ APPEND(CommonStatisticNames.OP_APPEND),
+ CONCAT("op_concat"),
+ COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE),
+ CREATE(CommonStatisticNames.OP_CREATE),
+ CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE),
+ CREATE_SNAPSHOT("op_create_snapshot"),
+ CREATE_SYM_LINK("op_create_symlink"),
+ DELETE(CommonStatisticNames.OP_DELETE),
+ DELETE_SNAPSHOT("op_delete_snapshot"),
+ DISALLOW_SNAPSHOT("op_disallow_snapshot"),
+ EXISTS(CommonStatisticNames.OP_EXISTS),
+ GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
+ GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
+ GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
+ GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
+ GET_FILE_LINK_STATUS("op_get_file_link_status"),
+ GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS),
+ GET_LINK_TARGET("op_get_link_target"),
+ GET_QUOTA_USAGE("op_get_quota_usage"),
+ GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
+ GET_STORAGE_POLICIES("op_get_storage_policies"),
+ GET_STORAGE_POLICY("op_get_storage_policy"),
+ GET_XATTR("op_get_xattr"),
+ LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
+ LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
+ MKDIRS(CommonStatisticNames.OP_MKDIRS),
+ MODIFY_ACL_ENTRIES(CommonStatisticNames.OP_MODIFY_ACL_ENTRIES),
+ OPEN(CommonStatisticNames.OP_OPEN),
+ PRIMITIVE_CREATE("op_primitive_create"),
+ PRIMITIVE_MKDIR("op_primitive_mkdir"),
+ REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
+ REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
+ REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
+ REMOVE_XATTR("op_remove_xattr"),
+ RENAME(CommonStatisticNames.OP_RENAME),
+ RENAME_SNAPSHOT("op_rename_snapshot"),
+ RESOLVE_LINK("op_resolve_link"),
+ SET_ACL(CommonStatisticNames.OP_SET_ACL),
+ SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
+ SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
+ SET_REPLICATION("op_set_replication"),
+ SET_STORAGE_POLICY("op_set_storagePolicy"),
+ SET_TIMES(CommonStatisticNames.OP_SET_TIMES),
+ SET_XATTR("op_set_xattr"),
+ TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
+ UNSET_STORAGE_POLICY("op_unset_storage_policy");
private final String symbol;
@@ -150,6 +151,11 @@ public class DFSOpsCountStatistics extends StorageStatistics {
}
@Override
+ public String getScheme() {
+ return HdfsConstants.HDFS_URI_SCHEME;
+ }
+
+ @Override
public Iterator<LongStatistic> getLongStatistics() {
return new LongIterator();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
index 7c1b018..7578930 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
@@ -31,8 +31,10 @@ import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -66,6 +68,19 @@ public class TestDFSOpsCountStatistics {
}
}
+ /**
+ * This is to test the the {@link OpType} symbols are unique.
+ */
+ @Test
+ public void testOpTypeSymbolsAreUnique() {
+ final Set<String> opTypeSymbols = new HashSet<>();
+ for (OpType opType : OpType.values()) {
+ assertFalse(opTypeSymbols.contains(opType.getSymbol()));
+ opTypeSymbols.add(opType.getSymbol());
+ }
+ assertEquals(OpType.values().length, opTypeSymbols.size());
+ }
+
@Test
public void testGetLongStatistics() {
short iterations = 0; // number of the iter.hasNext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
index a74b864..3a90c6b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
@@ -87,6 +87,11 @@ public class S3AStorageStatistics extends StorageStatistics {
}
@Override
+ public String getScheme() {
+ return "s3a";
+ }
+
+ @Override
public Iterator<LongStatistic> getLongStatistics() {
return new LongIterator();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index 8121b3b..3c205f3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.s3a;
+import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
+
/**
* Statistic which are collected in S3A.
* These statistics are available at a low level in {@link S3AStorageStatistics}
@@ -38,27 +40,27 @@ public enum Statistic {
FILES_DELETED("files_deleted",
"Total number of files deleted from the object store."),
IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"),
- INVOCATION_COPY_FROM_LOCAL_FILE("invocations_copyfromlocalfile",
+ INVOCATION_COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE,
"Calls of copyFromLocalFile()"),
- INVOCATION_EXISTS("invocations_exists",
+ INVOCATION_EXISTS(CommonStatisticNames.OP_EXISTS,
"Calls of exists()"),
- INVOCATION_GET_FILE_STATUS("invocations_getfilestatus",
+ INVOCATION_GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS,
"Calls of getFileStatus()"),
- INVOCATION_GLOB_STATUS("invocations_globstatus",
+ INVOCATION_GLOB_STATUS(CommonStatisticNames.OP_GLOB_STATUS,
"Calls of globStatus()"),
- INVOCATION_IS_DIRECTORY("invocations_is_directory",
+ INVOCATION_IS_DIRECTORY(CommonStatisticNames.OP_IS_DIRECTORY,
"Calls of isDirectory()"),
- INVOCATION_IS_FILE("invocations_is_file",
+ INVOCATION_IS_FILE(CommonStatisticNames.OP_IS_FILE,
"Calls of isFile()"),
- INVOCATION_LIST_FILES("invocations_listfiles",
+ INVOCATION_LIST_FILES(CommonStatisticNames.OP_LIST_FILES,
"Calls of listFiles()"),
- INVOCATION_LIST_LOCATED_STATUS("invocations_listlocatedstatus",
+ INVOCATION_LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS,
"Calls of listLocatedStatus()"),
- INVOCATION_LIST_STATUS("invocations_liststatus",
+ INVOCATION_LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS,
"Calls of listStatus()"),
- INVOCATION_MKDIRS("invocations_mdkirs",
+ INVOCATION_MKDIRS(CommonStatisticNames.OP_MKDIRS,
"Calls of mkdirs()"),
- INVOCATION_RENAME("invocations_rename",
+ INVOCATION_RENAME(CommonStatisticNames.OP_RENAME,
"Calls of rename()"),
OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"),
OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org