You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:27:04 UTC
svn commit: r1077540 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
hdfs/org/apache/hadoop/hdfs/server/namenode/
hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/
test/org/apache/hadoop/hdfs/server/namenode/metrics/
Author: omalley
Date: Fri Mar 4 04:27:04 2011
New Revision: 1077540
URL: http://svn.apache.org/viewvc?rev=1077540&view=rev
Log:
commit e33ffc0d1c61392536654415e024ff43e13c14e9
Author: Konstantin Shvachko <sh...@cdev6023.inktomisearch.com>
Date: Tue Jul 13 00:58:12 2010 +0000
HDFS:132 from https://issues.apache.org/jira/secure/attachment/12449291/HDFS-132.y20.patch
+++ b/YAHOO-CHANGES.txt
+ HDFS-132. Fix namenode to not report files deleted metrics for deletions
+ done while replaying edits during startup. (shv)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1077540&r1=1077539&r2=1077540&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Fri Mar 4 04:27:04 2011
@@ -24,9 +24,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.permission.*;
-import org.apache.hadoop.metrics.MetricsRecord;
-import org.apache.hadoop.metrics.MetricsUtil;
-import org.apache.hadoop.metrics.MetricsContext;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -52,8 +49,6 @@ class FSDirectory implements FSConstants
final INodeDirectoryWithQuota rootDir;
FSImage fsImage;
private boolean ready = false;
- // Metrics record
- private MetricsRecord directoryMetrics = null;
private final int lsLimit; // max list limit
/**
@@ -87,13 +82,6 @@ class FSDirectory implements FSConstants
+ " times ");
nameCache = new NameCache<ByteArray>(threshold);
- initialize(conf);
- }
-
- private void initialize(Configuration conf) {
- MetricsContext metricsContext = MetricsUtil.getContext("dfs");
- directoryMetrics = MetricsUtil.createRecord(metricsContext, "FSDirectory");
- directoryMetrics.setTag("sessionId", conf.get("session.id"));
}
void loadFSImage(Collection<File> dataDirs,
@@ -126,8 +114,8 @@ class FSDirectory implements FSConstants
}
private void incrDeletedFileCount(int count) {
- directoryMetrics.incrMetric("files_deleted", count);
- directoryMetrics.update();
+ if (namesystem != null)
+ NameNode.getNameNodeMetrics().numFilesDeleted.inc(count);
}
/**
@@ -593,17 +581,19 @@ class FSDirectory implements FSConstants
/**
* Remove the file from management, return blocks
*/
- INode delete(String src) {
+ boolean delete(String src) {
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: "+src);
}
waitForReady();
long now = FSNamesystem.now();
- INode deletedNode = unprotectedDelete(src, now);
- if (deletedNode != null) {
- fsImage.getEditLog().logDelete(src, now);
+ int filesRemoved = unprotectedDelete(src, now);
+ if (filesRemoved <= 0) {
+ return false;
}
- return deletedNode;
+ incrDeletedFileCount(filesRemoved);
+ fsImage.getEditLog().logDelete(src, now);
+ return true;
}
/** Return if a directory is empty or not **/
@@ -628,9 +618,9 @@ class FSDirectory implements FSConstants
* @param src a string representation of a path to an inode
* @param modificationTime the time the inode is removed
* @param deletedBlocks the place holder for the blocks to be removed
- * @return if the deletion succeeds
+ * @return the number of inodes deleted; 0 if no inodes are deleted.
*/
- INode unprotectedDelete(String src, long modificationTime) {
+ int unprotectedDelete(String src, long modificationTime) {
src = normalizePath(src);
synchronized (rootDir) {
@@ -640,12 +630,12 @@ class FSDirectory implements FSConstants
if (targetNode == null) { // non-existent src
NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
+"failed to remove "+src+" because it does not exist");
- return null;
+ return 0;
} else if (inodes.length == 1) { // src is the root
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedDelete: " +
"failed to remove " + src +
" because the root is not allowed to be deleted");
- return null;
+ return 0;
} else {
try {
// Remove the node from the namespace
@@ -655,17 +645,16 @@ class FSDirectory implements FSConstants
// GC all the blocks underneath the node.
ArrayList<Block> v = new ArrayList<Block>();
int filesRemoved = targetNode.collectSubtreeBlocksAndClear(v);
- incrDeletedFileCount(filesRemoved);
namesystem.removePathAndBlocks(src, v);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
+src+" is removed");
}
- return targetNode;
+ return filesRemoved;
} catch (IOException e) {
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedDelete: " +
"failed to remove " + src + " because " + e.getMessage());
- return null;
+ return 0;
}
}
}
@@ -986,7 +975,7 @@ class FSDirectory implements FSConstants
return false;
}
// Directory creation also count towards FilesCreated
- // to match count of files_deleted metric.
+ // to match count of FilesDeleted metric.
if (namesystem != null)
NameNode.getNameNodeMetrics().numFilesCreated.inc();
fsImage.getEditLog().logMkDir(cur, inodes[i]);
@@ -1210,7 +1199,6 @@ class FSDirectory implements FSConstants
* @param dir the root of the tree that represents the directory
* @param counters counters for name space and disk space
* @param nodesInPath INodes for the each of components in the path.
- * @return the size of the tree
*/
private static void updateCountForINodeWithQuota(INodeDirectory dir,
INode.DirCounts counts,
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1077540&r1=1077539&r2=1077540&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Mar 4 04:27:04 2011
@@ -1813,7 +1813,7 @@ public class FSNamesystem implements FSC
checkPermission(src, false, null, FsAction.WRITE, null, FsAction.ALL);
}
- return dir.delete(src) != null;
+ return dir.delete(src);
}
void removePathAndBlocks(String src, List<Block> blocks) throws IOException {
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java?rev=1077540&r1=1077539&r2=1077540&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java Fri Mar 4 04:27:04 2011
@@ -48,6 +48,8 @@ public class NameNodeMetrics implements
private NameNodeActivtyMBean namenodeActivityMBean;
+ public MetricsTimeVaryingInt numCreateFileOps =
+ new MetricsTimeVaryingInt("CreateFileOps", registry);
public MetricsTimeVaryingInt numFilesCreated =
new MetricsTimeVaryingInt("FilesCreated", registry);
public MetricsTimeVaryingInt numFilesAppended =
@@ -58,21 +60,23 @@ public class NameNodeMetrics implements
new MetricsTimeVaryingInt("FilesRenamed", registry);
public MetricsTimeVaryingInt numGetListingOps =
new MetricsTimeVaryingInt("GetListingOps", registry);
- public MetricsTimeVaryingInt numCreateFileOps =
- new MetricsTimeVaryingInt("CreateFileOps", registry);
public MetricsTimeVaryingInt numDeleteFileOps =
new MetricsTimeVaryingInt("DeleteFileOps", registry);
+ public MetricsTimeVaryingInt numFilesDeleted = new MetricsTimeVaryingInt(
+ "FilesDeleted", registry,
+ "Number of files and directories deleted by delete or rename operation");
public MetricsTimeVaryingInt numFileInfoOps =
new MetricsTimeVaryingInt("FileInfoOps", registry);
public MetricsTimeVaryingInt numAddBlockOps =
new MetricsTimeVaryingInt("AddBlockOps", registry);
- public MetricsTimeVaryingRate transactions =
- new MetricsTimeVaryingRate("Transactions", registry, "Journal Transaction");
+ public MetricsTimeVaryingRate transactions = new MetricsTimeVaryingRate(
+ "Transactions", registry, "Journal Transaction");
public MetricsTimeVaryingRate syncs =
new MetricsTimeVaryingRate("Syncs", registry, "Journal Sync");
- public MetricsTimeVaryingInt transactionsBatchedInSync =
- new MetricsTimeVaryingInt("JournalTransactionsBatchedInSync", registry, "Journal Transactions Batched In Sync");
+ public MetricsTimeVaryingInt transactionsBatchedInSync = new MetricsTimeVaryingInt(
+ "JournalTransactionsBatchedInSync", registry,
+ "Journal Transactions Batched In Sync");
public MetricsTimeVaryingRate blockReport =
new MetricsTimeVaryingRate("blockReport", registry, "Block Report");
public MetricsIntValue safeModeTime =
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java?rev=1077540&r1=1077539&r2=1077540&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java Fri Mar 4 04:27:04 2011
@@ -32,7 +32,6 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.mortbay.log.Log;
/**
* Test for metrics published by the Namenode
@@ -58,7 +57,6 @@ public class TestNameNodeMetrics extends
private Random rand = new Random();
private FSNamesystem namesystem;
private NameNodeMetrics nnMetrics;
- private NameNode nn;
private static Path getTestPath(String fileName) {
return new Path(TEST_ROOT_DIR_PATH, fileName);
@@ -71,8 +69,7 @@ public class TestNameNodeMetrics extends
namesystem = cluster.getNameNode().getNamesystem();
fs = (DistributedFileSystem) cluster.getFileSystem();
metrics = namesystem.getFSNamesystemMetrics();
- nn = cluster.getNameNode();
- nnMetrics = nn.getNameNodeMetrics();
+ nnMetrics = NameNode.getNameNodeMetrics();
}
@Override
@@ -90,19 +87,14 @@ public class TestNameNodeMetrics extends
// for some block related metrics to get updated)
Thread.sleep(1000);
metrics.doUpdates(null);
- }
-
- private void updateNNMetrics() throws Exception {
- //Wait for nnmetrics update
- Thread.sleep(1000);
nnMetrics.doUpdates(null);
}
-
+
private void readFile(FileSystem fileSys,Path name) throws IOException {
//Just read file so that getNumBlockLocations are incremented
DataInputStream stm = fileSys.open(name);
byte [] buffer = new byte[4];
- int bytesRead = stm.read(buffer,0,4);
+ stm.read(buffer,0,4);
stm.close();
}
@@ -115,6 +107,11 @@ public class TestNameNodeMetrics extends
int blockCapacity = namesystem.getBlockCapacity();
updateMetrics();
assertEquals(blockCapacity, metrics.blockCapacity.get());
+
+ // File create operations is 1
+ // Number of files created is depth of <code>file</code> path
+ assertEquals(1, nnMetrics.numCreateFileOps.getPreviousIntervalValue());
+ assertEquals(file.depth(), nnMetrics.numFilesCreated.getPreviousIntervalValue());
// Blocks are stored in a hashmap. Compute its capacity, which
// doubles every time the number of entries reach the threshold.
@@ -136,6 +133,10 @@ public class TestNameNodeMetrics extends
updateMetrics();
assertEquals(filesTotal, metrics.filesTotal.get());
assertEquals(0, metrics.pendingDeletionBlocks.get());
+
+ // Delete file operations and number of files deleted must be 1
+ assertEquals(1, nnMetrics.numDeleteFileOps.getPreviousIntervalValue());
+ assertEquals(1, nnMetrics.numFilesDeleted.getPreviousIntervalValue());
}
/** Corrupt a block and ensure metrics reflects it */
@@ -201,9 +202,6 @@ public class TestNameNodeMetrics extends
* @throws IOException in case of an error
*/
public void testGetBlockLocationMetric() throws Exception{
- final String METHOD_NAME = "TestGetBlockLocationMetric";
- Log.info("Running test "+METHOD_NAME);
-
Path file1_Path = new Path(TEST_ROOT_DIR_PATH, "file1.dat");
// When cluster starts first time there are no file (read,create,open)
@@ -217,8 +215,7 @@ public class TestNameNodeMetrics extends
//Perform create file operation
createFile(file1_Path,100,(short)2);
- // Update NameNode metrics
- updateNNMetrics();
+ updateMetrics();
//Create file does not change numGetBlockLocations metric
//expect numGetBlockLocations = 0 for previous and current interval
@@ -231,8 +228,7 @@ public class TestNameNodeMetrics extends
// Open and read file operation increments numGetBlockLocations
// Perform read file operation on earlier created file
readFile(fs, file1_Path);
- // Update NameNode metrics
- updateNNMetrics();
+ updateMetrics();
// Verify read file operation has incremented numGetBlockLocations by 1
assertEquals("numGetBlockLocations for previous interval is incorrect",
1,nnMetrics.numGetBlockLocations.getPreviousIntervalValue());
@@ -243,7 +239,7 @@ public class TestNameNodeMetrics extends
// opening and reading file twice will increment numGetBlockLocations by 2
readFile(fs, file1_Path);
readFile(fs, file1_Path);
- updateNNMetrics();
+ updateMetrics();
assertEquals("numGetBlockLocations for previous interval is incorrect",
2,nnMetrics.numGetBlockLocations.getPreviousIntervalValue());
// Verify numGetBlockLocations for current interval is 0