You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/05/12 23:21:03 UTC
svn commit: r1102466 - in /hadoop/common/branches/branch-0.20-security-204:
./ src/core/org/apache/hadoop/fs/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/mapred/
src/test/org/apache/hadoop/fs/ src/test/org/apache/hadoop/hdfs/
src/tools/org/apa...
Author: szetszwo
Date: Thu May 12 21:21:03 2011
New Revision: 1102466
URL: http://svn.apache.org/viewvc?rev=1102466&view=rev
Log:
HDFS-1377. svn merge -c 1100336 from branch-0.20-security
Modified:
hadoop/common/branches/branch-0.20-security-204/ (props changed)
hadoop/common/branches/branch-0.20-security-204/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/HarFileSystem.java (props changed)
hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/branch-0.20-security-204/src/mapred/ (props changed)
hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/fs/TestHarFileSystem.java (props changed)
hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/hdfs/TestQuota.java
hadoop/common/branches/branch-0.20-security-204/src/tools/org/apache/hadoop/tools/HadoopArchives.java (props changed)
Propchange: hadoop/common/branches/branch-0.20-security-204/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 12 21:21:03 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.20:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946
-/hadoop/common/branches/branch-0.20-security:1097202,1098837
+/hadoop/common/branches/branch-0.20:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
+/hadoop/common/branches/branch-0.20-security:1097202,1098837,1100336
/hadoop/common/branches/branch-0.20-security-203:1096071,1097012-1099333,1102071
/hadoop/core/branches/branch-0.19:713112
/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569
Modified: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/CHANGES.txt?rev=1102466&r1=1102465&r2=1102466&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-204/CHANGES.txt Thu May 12 21:21:03 2011
@@ -1,5 +1,9 @@
Hadoop Change Log
+Release 0.20.205.0 - unreleased
+
+ BUG FIXES
+
Release 0.20.204.0 - unreleased
BUG FIXES
@@ -56,6 +60,8 @@ Release 0.20.204.0 - unreleased
by throwing an error to indicate the editlog needs to be empty.
(suresh)
+ HDFS-1377. Quota bug for partial blocks allows quotas to be violated. (eli)
+
IMPROVEMENTS
MAPREDUCE-2415. Distribute the user task logs on to multiple disks.
Propchange: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 12 21:21:03 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946
-/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837
+/hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
+/hadoop/common/branches/branch-0.20-security/CHANGES.txt:1097202,1098837,1100336
/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097012-1099333,1102071
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
Propchange: hadoop/common/branches/branch-0.20-security-204/src/core/org/apache/hadoop/fs/HarFileSystem.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 12 21:21:03 2011
@@ -1,4 +1,5 @@
/hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/fs/HarFileSystem.java:826138,826568,833001,880632,898713
+/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/HarFileSystem.java:1100336
/hadoop/common/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java:910709
/hadoop/common/trunk/src/java/org/apache/hadoop/fs/HarFileSystem.java:979485
/hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/fs/HarFileSystem.java:713112
Modified: hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1102466&r1=1102465&r2=1102466&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu May 12 21:21:03 2011
@@ -663,19 +663,9 @@ class FSDirectory implements FSConstants
/**
* Replaces the specified inode with the specified one.
*/
- void replaceNode(String path, INodeFile oldnode, INodeFile newnode)
- throws IOException {
- replaceNode(path, oldnode, newnode, true);
- }
-
- /**
- * @see #replaceNode(String, INodeFile, INodeFile)
- */
- private void replaceNode(String path, INodeFile oldnode, INodeFile newnode,
- boolean updateDiskspace) throws IOException {
+ void replaceNode(String path, INodeFile oldnode, INodeFile newnode)
+ throws IOException {
synchronized (rootDir) {
- long dsOld = oldnode.diskspaceConsumed();
-
//
// Remove the node from the namespace
//
@@ -692,18 +682,6 @@ class FSDirectory implements FSConstants
rootDir.addNode(path, newnode);
- //check if disk space needs to be updated.
- long dsNew = 0;
- if (updateDiskspace && (dsNew = newnode.diskspaceConsumed()) != dsOld) {
- try {
- updateSpaceConsumed(path, 0, dsNew-dsOld);
- } catch (QuotaExceededException e) {
- // undo
- replaceNode(path, newnode, oldnode, false);
- throw e;
- }
- }
-
int index = 0;
for (Block b : newnode.getBlocks()) {
BlockInfo info = namesystem.blocksMap.addINode(b, newnode);
Modified: hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1102466&r1=1102465&r2=1102466&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Thu May 12 21:21:03 2011
@@ -338,6 +338,15 @@ class INodeDirectory extends INode {
child.computeContentSummary(summary);
}
}
+ if (this instanceof INodeDirectoryWithQuota) {
+ // Warn if the cached and computed diskspace values differ
+ INodeDirectoryWithQuota node = (INodeDirectoryWithQuota)this;
+ long space = node.diskspaceConsumed();
+ if (-1 != node.getDsQuota() && space != summary[3]) {
+ NameNode.LOG.warn("Inconsistent diskspace for directory "
+ +getLocalName()+". Cached: "+space+" Computed: "+summary[3]);
+ }
+ }
summary[2]++;
return summary;
}
Propchange: hadoop/common/branches/branch-0.20-security-204/src/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 12 21:21:03 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.20/src/mapred:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946
-/hadoop/common/branches/branch-0.20-security/src/mapred:1097202,1098837
+/hadoop/common/branches/branch-0.20/src/mapred:826138,826568,829987,831184,833001,880632,898713,909245,909723,960946,1044225
+/hadoop/common/branches/branch-0.20-security/src/mapred:1097202,1098837,1100336
/hadoop/common/branches/branch-0.20-security-203/src/mapred:1096071,1097012-1099333
/hadoop/core/branches/branch-0.19/src/mapred:713112
/hadoop/core/trunk/src/mapred:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569
Propchange: hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/fs/TestHarFileSystem.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 12 21:21:03 2011
@@ -1,4 +1,5 @@
/hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:826138,826568,833001,880632,898713
+/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:1100336
/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestHarFileSystem.java:979485
/hadoop/common/trunk/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:910709
/hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:713112
Modified: hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/hdfs/TestQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/hdfs/TestQuota.java?rev=1102466&r1=1102465&r2=1102466&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/hdfs/TestQuota.java (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/test/org/apache/hadoop/hdfs/TestQuota.java Thu May 12 21:21:03 2011
@@ -695,4 +695,124 @@ public class TestQuota extends TestCase
cluster.shutdown();
}
}
+
+ /**
+ * Violate a space quota using files of size < 1 block. Test that
+ * block allocation conservatively assumes that for quota checking
+ * the entire space of the block is used.
+ */
+ public void testBlockAllocationAdjustUsageConservatively() throws Exception {
+ Configuration conf = new Configuration();
+ final int BLOCK_SIZE = 6 * 1024;
+ conf.set("dfs.block.size", Integer.toString(BLOCK_SIZE));
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 3, true, null);
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+ DFSAdmin admin = new DFSAdmin(conf);
+
+ try {
+ Path dir = new Path("/test");
+ Path file1 = new Path("/test/test1");
+ Path file2 = new Path("/test/test2");
+ boolean exceededQuota = false;
+ final int QUOTA_SIZE = 3 * BLOCK_SIZE; // total usage including repl.
+ final int FILE_SIZE = BLOCK_SIZE / 2;
+ ContentSummary c;
+
+ // Create the directory and set the quota
+ assertTrue(fs.mkdirs(dir));
+ runCommand(admin, false, "-setSpaceQuota", Integer.toString(QUOTA_SIZE),
+ dir.toString());
+
+ // Creating one file should use half the quota
+ DFSTestUtil.createFile(fs, file1, FILE_SIZE, (short)3, 1L);
+ DFSTestUtil.waitReplication(fs, file1, (short)3);
+ c = fs.getContentSummary(dir);
+ assertEquals("Quota is half consumed", QUOTA_SIZE / 2,
+ c.getSpaceConsumed());
+
+ // We can not create the 2nd file because even though the total
+ // spaced used by two files (2 * 3 * 512/2) would fit within the
+ // quota (3 * 512) when a block for a file is created the space
+ // used is adjusted conservatively (3 * block size, ie assumes a
+ // full block is written) which will violate the quota (3 *
+ // block size) since we've already used half the quota for the
+ // first file.
+ try {
+ DFSTestUtil.createFile(fs, file2, FILE_SIZE, (short)3, 1L);
+ } catch (QuotaExceededException e) {
+ exceededQuota = true;
+ }
+ assertTrue("Quota not exceeded", exceededQuota);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Like the previous test but create many files. This covers bugs
+ * where the quota adjustment is incorrect but it takes many files
+ * to accrue a big enough accounting error to violate the quota.
+ */
+ public void testMultipleFilesSmallerThanOneBlock() throws Exception {
+ Configuration conf = new Configuration();
+ final int BLOCK_SIZE = 6 * 1024;
+ conf.set("dfs.block.size", Integer.toString(BLOCK_SIZE));
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 3, true, null);
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+ DFSAdmin admin = new DFSAdmin(conf);
+
+ try {
+ Path dir = new Path("/test");
+ boolean exceededQuota = false;
+ ContentSummary c;
+ // 1kb file
+ // 6kb block
+ // 192kb quota
+ final int FILE_SIZE = 1024;
+ final int QUOTA_SIZE = 32 * (int)fs.getDefaultBlockSize();
+ assertEquals(6 * 1024, fs.getDefaultBlockSize());
+ assertEquals(192 * 1024, QUOTA_SIZE);
+
+ // Create the dir and set the quota. We need to enable the quota before
+ // writing the files as setting the quota afterwards will over-write
+ // the cached disk space used for quota verification with the actual
+ // amount used as calculated by INode#spaceConsumedInTree.
+ assertTrue(fs.mkdirs(dir));
+ runCommand(admin, false, "-setSpaceQuota", Integer.toString(QUOTA_SIZE),
+ dir.toString());
+
+ // We can create at most 59 files because block allocation is
+ // conservative and initially assumes a full block is used, so we
+ // need to leave at least 3 * BLOCK_SIZE free space when allocating
+ // the last block: (58 * 3 * 1024) + (3 * 6 * 1024) = 192kb
+ for (int i = 0; i < 59; i++) {
+ Path file = new Path("/test/test" + i);
+ DFSTestUtil.createFile(fs, file, FILE_SIZE, (short)3, 1L);
+ DFSTestUtil.waitReplication(fs, file, (short)3);
+ }
+
+ // Should account for all 59 files (almost QUOTA_SIZE)
+ c = fs.getContentSummary(dir);
+ assertEquals("Invalid space consumed",
+ 59 * FILE_SIZE * 3,
+ c.getSpaceConsumed());
+ assertEquals("Invalid space consumed",
+ QUOTA_SIZE - (59 * FILE_SIZE * 3),
+ 3 * (fs.getDefaultBlockSize() - FILE_SIZE));
+
+ // Now check that trying to create another file violates the quota
+ try {
+ Path file = new Path("/test/test59");
+ DFSTestUtil.createFile(fs, file, FILE_SIZE, (short) 3, 1L);
+ DFSTestUtil.waitReplication(fs, file, (short) 3);
+ } catch (QuotaExceededException e) {
+ exceededQuota = true;
+ }
+ assertTrue("Quota not exceeded", exceededQuota);
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
Propchange: hadoop/common/branches/branch-0.20-security-204/src/tools/org/apache/hadoop/tools/HadoopArchives.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 12 21:21:03 2011
@@ -1,4 +1,5 @@
/hadoop/common/branches/branch-0.20/src/tools/org/apache/hadoop/tools/HadoopArchives.java:826138,826568,833001,880632,898713
+/hadoop/common/branches/branch-0.20-security/src/tools/org/apache/hadoop/tools/HadoopArchives.java:1100336
/hadoop/common/trunk/src/tools/org/apache/hadoop/tools/HadoopArchives.java:910709
/hadoop/core/branches/branch-0.19/src/tools/org/apache/hadoop/tools/HadoopArchives.java:713112
/hadoop/core/trunk/src/tools/org/apache/hadoop/tools/HadoopArchives.java:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569