You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/04/02 06:07:03 UTC
[hadoop] 02/05: HDFS-15266. Add missing DFSOps Statistics in
WebHDFS. Contributed by Ayush Saxena.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 5187bd37ae9c38dc55bb1e0451064a8f191cfca0
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Fri Apr 17 00:11:02 2020 +0530
HDFS-15266. Add missing DFSOps Statistics in WebHDFS. Contributed by Ayush Saxena.
(cherry picked from commit 37d65822235fe8285d10232589aba39ededd3be1)
---
.../apache/hadoop/hdfs/DistributedFileSystem.java | 13 +++--
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 8 +++
.../hadoop/hdfs/TestDistributedFileSystem.java | 21 ++++++--
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 60 ++++++++++++++++++++++
4 files changed, 96 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 53655d1..d585bbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -819,6 +819,8 @@ public class DistributedFileSystem extends FileSystem
@Override
public Collection<BlockStoragePolicy> getAllStoragePolicies()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES);
return Arrays.asList(dfs.getStoragePolicies());
}
@@ -840,9 +842,7 @@ public class DistributedFileSystem extends FileSystem
*/
@Deprecated
public BlockStoragePolicy[] getStoragePolicies() throws IOException {
- statistics.incrementReadOps(1);
- storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES);
- return dfs.getStoragePolicies();
+ return getAllStoragePolicies().toArray(new BlockStoragePolicy[0]);
}
/**
@@ -2162,6 +2162,9 @@ public class DistributedFileSystem extends FileSystem
*/
public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics
+ .incrementOpCounter(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST);
return dfs.getSnapshottableDirListing();
}
@@ -2334,6 +2337,8 @@ public class DistributedFileSystem extends FileSystem
*/
public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir,
final String fromSnapshot, final String toSnapshot) throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF);
Path absF = fixRelativePart(snapshotDir);
return new FileSystemLinkResolver<SnapshotDiffReport>() {
@Override
@@ -3282,6 +3287,8 @@ public class DistributedFileSystem extends FileSystem
*/
@Override
public Path getTrashRoot(Path path) {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT);
try {
if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
return super.getTrashRoot(path);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index ca59cf3..7a0857e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1341,6 +1341,8 @@ public class WebHdfsFileSystem extends FileSystem
@Override
public void satisfyStoragePolicy(final Path p) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.SATISFY_STORAGE_POLICY);
final HttpOpParam.Op op = PutOpParam.Op.SATISFYSTORAGEPOLICY;
new FsPathRunner(op, p).run();
}
@@ -1430,6 +1432,7 @@ public class WebHdfsFileSystem extends FileSystem
public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir,
final String fromSnapshot, final String toSnapshot) throws IOException {
+ statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF);
final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTDIFF;
return new FsPathResponseRunner<SnapshotDiffReport>(op, snapshotDir,
@@ -1444,6 +1447,7 @@ public class WebHdfsFileSystem extends FileSystem
public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList()
throws IOException {
+ statistics.incrementReadOps(1);
storageStatistics
.incrementOpCounter(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST);
final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTTABLEDIRECTORYLIST;
@@ -2005,6 +2009,8 @@ public class WebHdfsFileSystem extends FileSystem
@Override
public Collection<BlockStoragePolicy> getAllStoragePolicies()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES);
final HttpOpParam.Op op = GetOpParam.Op.GETALLSTORAGEPOLICY;
return new FsPathResponseRunner<Collection<BlockStoragePolicy>>(op, null) {
@Override
@@ -2017,6 +2023,8 @@ public class WebHdfsFileSystem extends FileSystem
@Override
public BlockStoragePolicy getStoragePolicy(Path src) throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICY);
final HttpOpParam.Op op = GetOpParam.Op.GETSTORAGEPOLICY;
return new FsPathResponseRunner<BlockStoragePolicy>(op, src) {
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 3c90356..e96bd62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -920,6 +920,21 @@ public class TestDistributedFileSystem {
dfs.getEZForPath(dir);
checkStatistics(dfs, ++readOps, writeOps, 0);
checkOpStatistics(OpType.GET_ENCRYPTION_ZONE, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST);
+ dfs.getSnapshottableDirListing();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_STORAGE_POLICIES);
+ dfs.getAllStoragePolicies();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_STORAGE_POLICIES, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_TRASH_ROOT);
+ dfs.getTrashRoot(dir);
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_TRASH_ROOT, opCount + 1);
}
}
@@ -1060,7 +1075,7 @@ public class TestDistributedFileSystem {
}
/** Checks statistics. -1 indicates do not check for the operations */
- private void checkStatistics(FileSystem fs, int readOps, int writeOps,
+ public static void checkStatistics(FileSystem fs, int readOps, int writeOps,
int largeReadOps) {
assertEquals(readOps, DFSTestUtil.getStatistics(fs).getReadOps());
assertEquals(writeOps, DFSTestUtil.getStatistics(fs).getWriteOps());
@@ -1166,12 +1181,12 @@ public class TestDistributedFileSystem {
}
}
- private static void checkOpStatistics(OpType op, long count) {
+ public static void checkOpStatistics(OpType op, long count) {
assertEquals("Op " + op.getSymbol() + " has unexpected count!",
count, getOpStatistics(op));
}
- private static long getOpStatistics(OpType op) {
+ public static long getOpStatistics(OpType op) {
return GlobalStorageStatistics.INSTANCE.get(
DFSOpsCountStatistics.NAME)
.getLong(op.getSymbol());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index d02a5fe..27a6985 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -24,6 +24,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkOpStatistics;
+import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkStatistics;
+import static org.apache.hadoop.hdfs.TestDistributedFileSystem.getOpStatistics;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
import static org.junit.Assert.assertEquals;
@@ -58,6 +61,7 @@ import java.util.Random;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.hdfs.DFSOpsCountStatistics;
import org.apache.hadoop.test.LambdaTestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2013,6 +2017,62 @@ public class TestWebHDFS {
ecpolicyForECfile, ecPolicyName);
}
+ @Test
+ public void testStatistics() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
+ StoragePolicySatisfierMode.EXTERNAL.toString());
+ StoragePolicySatisfier sps = new StoragePolicySatisfier(conf);
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).storageTypes(
+ new StorageType[][] {{StorageType.DISK, StorageType.ARCHIVE}})
+ .storagesPerDatanode(2).numDataNodes(1).build();
+ cluster.waitActive();
+ sps.init(new ExternalSPSContext(sps, DFSTestUtil
+ .getNameNodeConnector(conf, HdfsServerConstants.MOVER_ID_PATH, 1,
+ false)));
+ sps.start(StoragePolicySatisfierMode.EXTERNAL);
+ sps.start(StoragePolicySatisfierMode.EXTERNAL);
+ final WebHdfsFileSystem webHdfs = WebHdfsTestUtil
+ .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ Path dir = new Path("/test");
+ webHdfs.mkdirs(dir);
+ int readOps = 0;
+ int writeOps = 0;
+ FileSystem.clearStatistics();
+
+ long opCount =
+ getOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICY);
+ webHdfs.getStoragePolicy(dir);
+ checkStatistics(webHdfs, ++readOps, writeOps, 0);
+ checkOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICY,
+ opCount + 1);
+
+ opCount =
+ getOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICIES);
+ webHdfs.getAllStoragePolicies();
+ checkStatistics(webHdfs, ++readOps, writeOps, 0);
+ checkOpStatistics(DFSOpsCountStatistics.OpType.GET_STORAGE_POLICIES,
+ opCount + 1);
+
+ opCount =
+ getOpStatistics(DFSOpsCountStatistics.OpType.SATISFY_STORAGE_POLICY);
+ webHdfs.satisfyStoragePolicy(dir);
+ checkStatistics(webHdfs, readOps, ++writeOps, 0);
+ checkOpStatistics(DFSOpsCountStatistics.OpType.SATISFY_STORAGE_POLICY,
+ opCount + 1);
+
+ opCount = getOpStatistics(
+ DFSOpsCountStatistics.OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST);
+ webHdfs.getSnapshottableDirectoryList();
+ checkStatistics(webHdfs, ++readOps, writeOps, 0);
+ checkOpStatistics(
+ DFSOpsCountStatistics.OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST,
+ opCount + 1);
+ } finally {
+ cluster.shutdown();
+ }
+ }
/**
* Get FileStatus JSONObject from ListStatus response.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org