You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2013/12/06 02:30:22 UTC
svn commit: r1548359 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/
Author: jing9
Date: Fri Dec 6 01:30:22 2013
New Revision: 1548359
URL: http://svn.apache.org/r1548359
Log:
HDFS-5633. Improve OfflineImageViewer to use less memory. Contributed by Jing Zhao.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1548359&r1=1548358&r2=1548359&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Dec 6 01:30:22 2013
@@ -576,6 +576,8 @@ Release 2.4.0 - UNRELEASED
HDFS-5581. NameNodeFsck should use only one instance of
BlockPlacementPolicy. (vinay via cmccabe)
+ HDFS-5633. Improve OfflineImageViewer to use less memory. (jing9)
+
OPTIMIZATIONS
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java?rev=1548359&r1=1548358&r2=1548359&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java Fri Dec 6 01:30:22 2013
@@ -100,6 +100,18 @@ class FileDistributionVisitor extends Te
@Override
void finish() throws IOException {
+ output();
+ super.finish();
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ System.out.println("*** Image processing finished abnormally. Ending ***");
+ output();
+ super.finishAbnormally();
+ }
+
+ private void output() throws IOException {
// write the distribution into the output file
write("Size\tNumFiles\n");
for(int i = 0; i < distribution.length; i++)
@@ -109,7 +121,6 @@ class FileDistributionVisitor extends Te
System.out.println("totalBlocks = " + totalBlocks);
System.out.println("totalSpace = " + totalSpace);
System.out.println("maxFileSize = " + maxFileSize);
- super.finish();
}
@Override
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1548359&r1=1548358&r2=1548359&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Fri Dec 6 01:30:22 2013
@@ -129,7 +129,7 @@ class ImageLoaderCurrent implements Imag
-40, -41, -42, -43, -44, -45, -46, -47, -48 };
private int imageVersion = 0;
- private final Map<Long, String> subtreeMap = new HashMap<Long, String>();
+ private final Map<Long, Boolean> subtreeMap = new HashMap<Long, Boolean>();
private final Map<Long, String> dirNodeMap = new HashMap<Long, String>();
/* (non-Javadoc)
@@ -500,11 +500,15 @@ class ImageLoaderCurrent implements Imag
// 1. load dir node id
long inodeId = in.readLong();
- String dirName = dirNodeMap.get(inodeId);
- String oldValue = subtreeMap.put(inodeId, dirName);
- if (oldValue != null) { // the subtree has been visited
- return;
- }
+ String dirName = dirNodeMap.remove(inodeId);
+ Boolean visitedRef = subtreeMap.get(inodeId);
+ if (visitedRef != null) {
+ if (visitedRef.booleanValue()) { // the subtree has been visited
+ return;
+ } else { // first time to visit
+ subtreeMap.put(inodeId, true);
+ }
+ } // else the dir is not linked by a RefNode, thus cannot be revisited
// 2. load possible snapshots
processSnapshots(in, v, dirName);
@@ -695,6 +699,8 @@ class ImageLoaderCurrent implements Imag
if (numBlocks >= 0) { // File
if (supportSnapshot) {
+ // make sure subtreeMap only contains entry for directory
+ subtreeMap.remove(inodeId);
// process file diffs
processFileDiffList(in, v, parentName);
if (isSnapshotCopy) {
@@ -738,6 +744,11 @@ class ImageLoaderCurrent implements Imag
final boolean firstReferred = in.readBoolean();
if (firstReferred) {
+ // if a subtree is linked by multiple "parents", the corresponding dir
+ // must be referred by a reference node. we put the reference node into
+ // the subtreeMap here and let its value be false. when we later visit
+ // the subtree for the first time, we change the value to true.
+ subtreeMap.put(inodeId, false);
v.visitEnclosingElement(ImageElement.SNAPSHOT_REF_INODE);
processINode(in, v, skipBlocks, parentName, isSnapshotCopy);
v.leaveEnclosingElement(); // referred inode