You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2015/05/18 21:37:32 UTC
hadoop git commit: HDFS-4185. Add a metric for number of active
leases (Rakesh R via raviprak)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0c590e1c0 -> cdfae446a
HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdfae446
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdfae446
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdfae446
Branch: refs/heads/trunk
Commit: cdfae446ad285db979a79bf55665363fd943702c
Parents: 0c590e1
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Mon May 18 12:37:21 2015 -0700
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Mon May 18 12:37:21 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hdfs/server/namenode/FSNamesystem.java | 17 ++++++
.../hdfs/server/namenode/LeaseManager.java | 9 +++
.../namenode/metrics/TestNameNodeMetrics.java | 59 ++++++++++++++++++--
4 files changed, 83 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdfae446/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7fd3495..35c3b5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -570,6 +570,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8345. Storage policy APIs must be exposed via the FileSystem
interface. (Arpit Agarwal)
+ HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdfae446/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0fec5ee..7e5b981 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5347,6 +5347,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * Get the number of files under construction in the system.
+ */
+ @Metric({ "NumFilesUnderConstruction",
+ "Number of files under construction" })
+ public long getNumFilesUnderConstruction() {
+ return leaseManager.countPath();
+ }
+
+ /**
+ * Get the total number of active clients holding lease in the system.
+ */
+ @Metric({ "NumActiveClients", "Number of active clients holding lease" })
+ public long getNumActiveClients() {
+ return leaseManager.countLease();
+ }
+
+ /**
* Get the total number of COMPLETE blocks in the system.
* For safe mode only complete blocks are counted.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdfae446/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index ade2312..0806f82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -130,6 +130,15 @@ public class LeaseManager {
@VisibleForTesting
public synchronized int countLease() {return sortedLeases.size();}
+ /** @return the number of paths contained in all leases */
+ synchronized int countPath() {
+ int count = 0;
+ for (Lease lease : sortedLeases) {
+ count += lease.getFiles().size();
+ }
+ return count;
+ }
+
/**
* Adds (or re-adds) the lease for the specified file.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdfae446/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index b390391..3120f85 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode.metrics;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
@@ -29,12 +28,12 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.util.Random;
import com.google.common.collect.ImmutableList;
-import com.google.common.io.Files;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
@@ -55,12 +54,10 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
-import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.test.MetricsAsserts;
-import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
@@ -567,4 +564,58 @@ public class TestNameNodeMetrics {
assertTrue(MetricsAsserts.getLongCounter("TransactionsNumOps", rbNew) >
startWriteCounter);
}
+
+ /**
+ * Test metrics indicating the number of active clients and the files under
+ * construction
+ */
+ @Test(timeout = 60000)
+ public void testNumActiveClientsAndFilesUnderConstructionMetrics()
+ throws Exception {
+ final Path file1 = getTestPath("testFileAdd1");
+ createFile(file1, 100, (short) 3);
+ assertGauge("NumActiveClients", 0L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 0L, getMetrics(NS_METRICS));
+
+ Path file2 = new Path("/testFileAdd2");
+ FSDataOutputStream output2 = fs.create(file2);
+ output2.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 1L, getMetrics(NS_METRICS));
+
+ Path file3 = new Path("/testFileAdd3");
+ FSDataOutputStream output3 = fs.create(file3);
+ output3.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 2L, getMetrics(NS_METRICS));
+
+ // create another DistributedFileSystem client
+ DistributedFileSystem fs1 = (DistributedFileSystem) cluster
+ .getNewFileSystemInstance(0);
+ try {
+ Path file4 = new Path("/testFileAdd4");
+ FSDataOutputStream output4 = fs1.create(file4);
+ output4.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 2L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 3L, getMetrics(NS_METRICS));
+
+ Path file5 = new Path("/testFileAdd35");
+ FSDataOutputStream output5 = fs1.create(file5);
+ output5.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 2L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 4L, getMetrics(NS_METRICS));
+
+ output2.close();
+ output3.close();
+ assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 2L, getMetrics(NS_METRICS));
+
+ output4.close();
+ output5.close();
+ assertGauge("NumActiveClients", 0L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 0L, getMetrics(NS_METRICS));
+ } finally {
+ fs1.close();
+ }
+ }
}