You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2014/06/20 20:25:09 UTC
svn commit: r1604226 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
Author: jing9
Date: Fri Jun 20 18:25:09 2014
New Revision: 1604226
URL: http://svn.apache.org/r1604226
Log:
HDFS-6535. HDFS quota update is wrong when file is appended. Contributed by George Wong.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1604226&r1=1604225&r2=1604226&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jun 20 18:25:09 2014
@@ -679,6 +679,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS
client. (atm)
+ HDFS-6535. HDFS quota update is wrong when file is appended. (George Wong
+ via jing9)
+
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1604226&r1=1604225&r2=1604226&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Jun 20 18:25:09 2014
@@ -2512,7 +2512,7 @@ public class FSNamesystem implements Nam
if (ret != null) {
// update the quota: use the preferred block size for UC block
final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
- dir.updateSpaceConsumed(src, 0, diff);
+ dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication());
}
if (writeToEditLog) {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java?rev=1604226&r1=1604225&r2=1604226&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java Fri Jun 20 18:25:09 2014
@@ -18,10 +18,12 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -36,7 +38,9 @@ import org.junit.Test;
public class TestDiskspaceQuotaUpdate {
private static final int BLOCKSIZE = 1024;
- private static final short REPLICATION = 1;
+ private static final short REPLICATION = 4;
+ static final long seed = 0L;
+ private static final Path dir = new Path("/TestQuotaUpdate");
private Configuration conf;
private MiniDFSCluster cluster;
@@ -63,41 +67,83 @@ public class TestDiskspaceQuotaUpdate {
}
/**
+ * Test if the quota can be correctly updated for create file
+ */
+ @Test (timeout=60000)
+ public void testQuotaUpdateWithFileCreate() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ Path createdFile = new Path(foo, "created_file.data");
+ dfs.mkdirs(foo);
+ dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1);
+ long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16,
+ fileLen, BLOCKSIZE, REPLICATION, seed);
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+ Quota.Counts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed();
+ assertEquals(2, cnt.get(Quota.NAMESPACE));
+ assertEquals(fileLen * REPLICATION, cnt.get(Quota.DISKSPACE));
+ }
+
+ /**
* Test if the quota can be correctly updated for append
*/
- @Test
+ @Test (timeout=60000)
public void testUpdateQuotaForAppend() throws Exception {
- final Path foo = new Path("/foo");
+ final Path foo = new Path(dir ,"foo");
final Path bar = new Path(foo, "bar");
- DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L);
+ long currentFileLen = BLOCKSIZE;
+ DFSTestUtil.createFile(dfs, bar, currentFileLen, REPLICATION, seed);
dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1);
- // append half of the block data
+ // append half of the block data, the previous file length is at block
+ // boundary
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2);
+ currentFileLen += (BLOCKSIZE / 2);
INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
+ assertTrue(fooNode.isQuotaSet());
Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed();
long ns = quota.get(Quota.NAMESPACE);
long ds = quota.get(Quota.DISKSPACE);
assertEquals(2, ns); // foo and bar
- assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);
+ assertEquals(currentFileLen * REPLICATION, ds);
+ ContentSummary c = dfs.getContentSummary(foo);
+ assertEquals(c.getSpaceConsumed(), ds);
- // append another block
+ // append another block, the previous file length is not at block boundary
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);
+ currentFileLen += BLOCKSIZE;
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.get(Quota.NAMESPACE);
ds = quota.get(Quota.DISKSPACE);
assertEquals(2, ns); // foo and bar
- assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
+ assertEquals(currentFileLen * REPLICATION, ds);
+ c = dfs.getContentSummary(foo);
+ assertEquals(c.getSpaceConsumed(), ds);
+
+ // append several blocks
+ DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE * 3 + BLOCKSIZE / 8);
+ currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8);
+
+ quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
+ ns = quota.get(Quota.NAMESPACE);
+ ds = quota.get(Quota.DISKSPACE);
+ assertEquals(2, ns); // foo and bar
+ assertEquals(currentFileLen * REPLICATION, ds);
+ c = dfs.getContentSummary(foo);
+ assertEquals(c.getSpaceConsumed(), ds);
}
/**
* Test if the quota can be correctly updated when file length is updated
* through fsync
*/
- @Test
+ @Test (timeout=60000)
public void testUpdateQuotaForFSync() throws Exception {
final Path foo = new Path("/foo");
final Path bar = new Path(foo, "bar");