You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/11/14 17:55:32 UTC
svn commit: r1541975 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/ sr...
Author: kihwal
Date: Thu Nov 14 16:55:31 2013
New Revision: 1541975
URL: http://svn.apache.org/r1541975
Log:
svn merge -c 1541971 merging from trunk to branch-2 to fix HDFS-4995 Make getContentSummary less expensive.
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
- copied unchanged from r1541971, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Nov 14 16:55:31 2013
@@ -114,6 +114,8 @@ Release 2.3.0 - UNRELEASED
HDFS-5487. Introduce unit test for TokenAspect. (Haohui Mai via jing9)
+ HDFS-4995. Make getContentSummary less expensive. (kihwal)
+
OPTIMIZATIONS
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Nov 14 16:55:31 2013
@@ -192,6 +192,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
+ public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit";
+ public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 0;
public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated";
public static final int DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT = 0;
public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose";
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Nov 14 16:55:31 2013
@@ -112,7 +112,9 @@ public class FSDirectory implements Clos
private final int maxComponentLength;
private final int maxDirItems;
private final int lsLimit; // max list limit
+ private final int contentCountLimit; // max content summary counts per run
private final INodeMap inodeMap; // Synchronized by dirLock
+ private long yieldCount = 0; // keep track of lock yield count.
// lock to protect the directory and BlockMap
private ReentrantReadWriteLock dirLock;
@@ -143,6 +145,14 @@ public class FSDirectory implements Clos
return this.dirLock.getReadHoldCount() > 0;
}
+ public int getReadHoldCount() {
+ return this.dirLock.getReadHoldCount();
+ }
+
+ public int getWriteHoldCount() {
+ return this.dirLock.getWriteHoldCount();
+ }
+
/**
* Caches frequently used file names used in {@link INode} to reuse
* byte[] objects and reduce heap usage.
@@ -159,6 +169,10 @@ public class FSDirectory implements Clos
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
this.lsLimit = configuredLimit>0 ?
configuredLimit : DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT;
+
+ this.contentCountLimit = conf.getInt(
+ DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY,
+ DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_DEFAULT);
// filesystem limits
this.maxComponentLength = conf.getInt(
@@ -2294,13 +2308,26 @@ public class FSDirectory implements Clos
throw new FileNotFoundException("File does not exist: " + srcs);
}
else {
- return targetNode.computeContentSummary();
+ // Make it relinquish locks everytime contentCountLimit entries are
+ // processed. 0 means disabled. I.e. blocking for the entire duration.
+ ContentSummaryComputationContext cscc =
+
+ new ContentSummaryComputationContext(this, getFSNamesystem(),
+ contentCountLimit);
+ ContentSummary cs = targetNode.computeAndConvertContentSummary(cscc);
+ yieldCount += cscc.getYieldCount();
+ return cs;
}
} finally {
readUnlock();
}
}
+ @VisibleForTesting
+ public long getYieldCount() {
+ return yieldCount;
+ }
+
public INodeMap getINodeMap() {
return inodeMap;
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Nov 14 16:55:31 2013
@@ -1272,6 +1272,14 @@ public class FSNamesystem implements Nam
return hasReadLock() || hasWriteLock();
}
+ public int getReadHoldCount() {
+ return this.fsLock.getReadHoldCount();
+ }
+
+ public int getWriteHoldCount() {
+ return this.fsLock.getWriteHoldCount();
+ }
+
NamespaceInfo getNamespaceInfo() {
readLock();
try {
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Thu Nov 14 16:55:31 2013
@@ -371,10 +371,18 @@ public abstract class INode implements I
public abstract void destroyAndCollectBlocks(
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes);
- /** Compute {@link ContentSummary}. */
+ /** Compute {@link ContentSummary}. Blocking call */
public final ContentSummary computeContentSummary() {
- final Content.Counts counts = computeContentSummary(
- Content.Counts.newInstance());
+ return computeAndConvertContentSummary(
+ new ContentSummaryComputationContext());
+ }
+
+ /**
+ * Compute {@link ContentSummary}.
+ */
+ public final ContentSummary computeAndConvertContentSummary(
+ ContentSummaryComputationContext summary) {
+ Content.Counts counts = computeContentSummary(summary).getCounts();
return new ContentSummary(counts.get(Content.LENGTH),
counts.get(Content.FILE) + counts.get(Content.SYMLINK),
counts.get(Content.DIRECTORY), getNsQuota(),
@@ -384,10 +392,12 @@ public abstract class INode implements I
/**
* Count subtree content summary with a {@link Content.Counts}.
*
- * @param counts The subtree counts for returning.
- * @return The same objects as the counts parameter.
+ * @param summary the context object holding counts for the subtree.
+ * @return The same objects as summary.
*/
- public abstract Content.Counts computeContentSummary(Content.Counts counts);
+ public abstract ContentSummaryComputationContext computeContentSummary(
+ ContentSummaryComputationContext summary);
+
/**
* Check and add namespace/diskspace consumed to itself and the ancestors.
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Thu Nov 14 16:55:31 2013
@@ -466,12 +466,45 @@ public class INodeDirectory extends INod
}
@Override
- public Content.Counts computeContentSummary(final Content.Counts counts) {
- for (INode child : getChildrenList(null)) {
- child.computeContentSummary(counts);
+ public ContentSummaryComputationContext computeContentSummary(
+ ContentSummaryComputationContext summary) {
+ ReadOnlyList<INode> childrenList = getChildrenList(null);
+ // Explicit traversing is done to enable repositioning after relinquishing
+ // and reacquiring locks.
+ for (int i = 0; i < childrenList.size(); i++) {
+ INode child = childrenList.get(i);
+ byte[] childName = child.getLocalNameBytes();
+
+ long lastYieldCount = summary.getYieldCount();
+ child.computeContentSummary(summary);
+
+ // Check whether the computation was paused in the subtree.
+ // The counts may be off, but traversing the rest of children
+ // should be made safe.
+ if (lastYieldCount == summary.getYieldCount()) {
+ continue;
+ }
+
+ // The locks were released and reacquired. Check parent first.
+ if (getParent() == null) {
+ // Stop further counting and return whatever we have so far.
+ break;
+ }
+
+ // Obtain the children list again since it may have been modified.
+ childrenList = getChildrenList(null);
+ // Reposition in case the children list is changed. Decrement by 1
+ // since it will be incremented when loops.
+ i = nextChild(childrenList, childName) - 1;
}
- counts.add(Content.DIRECTORY, 1);
- return counts;
+
+ // Increment the directory count for this directory.
+ summary.getCounts().add(Content.DIRECTORY, 1);
+
+ // Relinquish and reacquire locks if necessary.
+ summary.yield();
+
+ return summary;
}
/**
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java Thu Nov 14 16:55:31 2013
@@ -107,12 +107,16 @@ public class INodeDirectoryWithQuota ext
}
@Override
- public Content.Counts computeContentSummary(
- final Content.Counts counts) {
- final long original = counts.get(Content.DISKSPACE);
- super.computeContentSummary(counts);
- checkDiskspace(counts.get(Content.DISKSPACE) - original);
- return counts;
+ public ContentSummaryComputationContext computeContentSummary(
+ final ContentSummaryComputationContext summary) {
+ final long original = summary.getCounts().get(Content.DISKSPACE);
+ long oldYieldCount = summary.getYieldCount();
+ super.computeContentSummary(summary);
+ // Check only when the content has not changed in the middle.
+ if (oldYieldCount == summary.getYieldCount()) {
+ checkDiskspace(summary.getCounts().get(Content.DISKSPACE) - original);
+ }
+ return summary;
}
private void checkDiskspace(final long computed) {
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Thu Nov 14 16:55:31 2013
@@ -344,11 +344,11 @@ public class INodeFile extends INodeWith
}
@Override
- public final Content.Counts computeContentSummary(
- final Content.Counts counts) {
- computeContentSummary4Snapshot(counts);
- computeContentSummary4Current(counts);
- return counts;
+ public final ContentSummaryComputationContext computeContentSummary(
+ final ContentSummaryComputationContext summary) {
+ computeContentSummary4Snapshot(summary.getCounts());
+ computeContentSummary4Current(summary.getCounts());
+ return summary;
}
private void computeContentSummary4Snapshot(final Content.Counts counts) {
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java Thu Nov 14 16:55:31 2013
@@ -107,7 +107,8 @@ public class INodeMap {
}
@Override
- public Content.Counts computeContentSummary(Content.Counts counts) {
+ public ContentSummaryComputationContext computeContentSummary(
+ ContentSummaryComputationContext summary) {
return null;
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java Thu Nov 14 16:55:31 2013
@@ -278,8 +278,9 @@ public abstract class INodeReference ext
}
@Override
- public Content.Counts computeContentSummary(Content.Counts counts) {
- return referred.computeContentSummary(counts);
+ public ContentSummaryComputationContext computeContentSummary(
+ ContentSummaryComputationContext summary) {
+ return referred.computeContentSummary(summary);
}
@Override
@@ -444,12 +445,13 @@ public abstract class INodeReference ext
}
@Override
- public final Content.Counts computeContentSummary(Content.Counts counts) {
+ public final ContentSummaryComputationContext computeContentSummary(
+ ContentSummaryComputationContext summary) {
//only count diskspace for WithName
final Quota.Counts q = Quota.Counts.newInstance();
computeQuotaUsage(q, false, lastSnapshotId);
- counts.add(Content.DISKSPACE, q.get(Quota.DISKSPACE));
- return counts;
+ summary.getCounts().add(Content.DISKSPACE, q.get(Quota.DISKSPACE));
+ return summary;
}
@Override
@@ -688,4 +690,4 @@ public abstract class INodeReference ext
}
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java Thu Nov 14 16:55:31 2013
@@ -98,9 +98,10 @@ public class INodeSymlink extends INodeW
}
@Override
- public Content.Counts computeContentSummary(final Content.Counts counts) {
- counts.add(Content.SYMLINK, 1);
- return counts;
+ public ContentSummaryComputationContext computeContentSummary(
+ final ContentSummaryComputationContext summary) {
+ summary.getCounts().add(Content.SYMLINK, 1);
+ return summary;
}
@Override
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Thu Nov 14 16:55:31 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.S
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.Content;
+import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@@ -342,11 +343,12 @@ public class INodeDirectorySnapshottable
}
@Override
- public Content.Counts computeContentSummary(final Content.Counts counts) {
- super.computeContentSummary(counts);
- counts.add(Content.SNAPSHOT, snapshotsByNames.size());
- counts.add(Content.SNAPSHOTTABLE_DIRECTORY, 1);
- return counts;
+ public ContentSummaryComputationContext computeContentSummary(
+ final ContentSummaryComputationContext summary) {
+ super.computeContentSummary(summary);
+ summary.getCounts().add(Content.SNAPSHOT, snapshotsByNames.size());
+ summary.getCounts().add(Content.SNAPSHOTTABLE_DIRECTORY, 1);
+ return summary;
}
/**
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java Thu Nov 14 16:55:31 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.Q
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.Content;
+import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
@@ -883,18 +884,27 @@ public class INodeDirectoryWithSnapshot
}
@Override
- public Content.Counts computeContentSummary(final Content.Counts counts) {
- super.computeContentSummary(counts);
- computeContentSummary4Snapshot(counts);
- return counts;
+ public ContentSummaryComputationContext computeContentSummary(
+ final ContentSummaryComputationContext summary) {
+ // Snapshot summary calc won't be relinquishing locks in the middle.
+ // Do this first and handover to parent.
+ computeContentSummary4Snapshot(summary.getCounts());
+ super.computeContentSummary(summary);
+ return summary;
}
private void computeContentSummary4Snapshot(final Content.Counts counts) {
+ // Create a new blank summary context for blocking processing of subtree.
+ ContentSummaryComputationContext summary =
+ new ContentSummaryComputationContext();
for(DirectoryDiff d : diffs) {
for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
- deleted.computeContentSummary(counts);
+ deleted.computeContentSummary(summary);
}
}
+ // Add the counts from deleted trees.
+ counts.add(summary.getCounts());
+ // Add the deleted directory count.
counts.add(Content.DIRECTORY, diffs.asList().size());
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java?rev=1541975&r1=1541974&r2=1541975&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java Thu Nov 14 16:55:31 2013
@@ -84,6 +84,9 @@ public class TestQuota {
// Space quotas
final int DEFAULT_BLOCK_SIZE = 512;
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+ // Make it relinquish locks. When run serially, the result should
+ // be identical.
+ conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
final FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: "+fs.getUri(),
@@ -348,6 +351,7 @@ public class TestQuota {
}
assertTrue(hasException);
+ assertEquals(4, cluster.getNamesystem().getFSDirectory().getYieldCount());
} finally {
cluster.shutdown();
}
@@ -358,6 +362,9 @@ public class TestQuota {
@Test
public void testNamespaceCommands() throws Exception {
final Configuration conf = new HdfsConfiguration();
+ // Make it relinquish locks. When run serially, the result should
+ // be identical.
+ conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
final FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: "+fs.getUri(),
@@ -513,6 +520,7 @@ public class TestQuota {
c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getDirectoryCount(), 6);
assertEquals(c.getQuota(), 6);
+ assertEquals(14, cluster.getNamesystem().getFSDirectory().getYieldCount());
} finally {
cluster.shutdown();
}
@@ -530,6 +538,9 @@ public class TestQuota {
// set a smaller block size so that we can test with smaller
// diskspace quotas
conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512");
+ // Make it relinquish locks. When run serially, the result should
+ // be identical.
+ conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
final FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: "+fs.getUri(),
@@ -762,6 +773,7 @@ public class TestQuota {
assertEquals(c.getSpaceConsumed(),
(sizeFactorA + sizeFactorB + sizeFactorC) * fileSpace);
+ assertEquals(20, cluster.getNamesystem().getFSDirectory().getYieldCount());
} finally {
cluster.shutdown();
}
@@ -846,6 +858,9 @@ public class TestQuota {
final int BLOCK_SIZE = 6 * 1024;
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+ // Make it relinquish locks. When run serially, the result should
+ // be identical.
+ conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
@@ -904,6 +919,7 @@ public class TestQuota {
exceededQuota = true;
}
assertTrue("Quota not exceeded", exceededQuota);
+ assertEquals(2, cluster.getNamesystem().getFSDirectory().getYieldCount());
} finally {
cluster.shutdown();
}