You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:19:40 UTC
svn commit: r1077483 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
test/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Author: omalley
Date: Fri Mar 4 04:19:39 2011
New Revision: 1077483
URL: http://svn.apache.org/viewvc?rev=1077483&view=rev
Log:
commit c25855da8b0a00dd518a66322ca71f0a750832c0
Author: Bharath Mundlapudi <bh...@yahoo-inc.com>
Date: Wed Jun 2 14:51:56 2010 -0700
HDFS:1061 from https://issues.apache.org/jira/secure/attachment/12446181/HDFS-1061-rel20.patch
+++ b/YAHOO-CHANGES.txt
+Release 0.20.1xx - unreleased
+
+ HDFS-1061. INodeFile memory optimization. (bharathm)
Added:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1077483&r1=1077482&r2=1077483&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Fri Mar 4 04:19:39 2011
@@ -29,9 +29,16 @@ import org.apache.hadoop.hdfs.server.nam
class INodeFile extends INode {
static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+ //Number of bits for Block size
+ static final short BLOCKBITS = 48;
+
+ //Header mask 64-bit representation
+ //Format: [16 bits for replication][48 bits for PreferredBlockSize]
+ static final long HEADERMASK = 0xffffL << BLOCKBITS;
+
+ protected long header;
+
protected BlockInfo blocks[] = null;
- protected short blockReplication;
- protected long preferredBlockSize;
INodeFile(PermissionStatus permissions,
int nrBlocks, short replication, long modificationTime,
@@ -42,16 +49,15 @@ class INodeFile extends INode {
protected INodeFile() {
blocks = null;
- blockReplication = 0;
- preferredBlockSize = 0;
+ header = 0;
}
protected INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
short replication, long modificationTime,
long atime, long preferredBlockSize) {
super(permissions, modificationTime, atime);
- this.blockReplication = replication;
- this.preferredBlockSize = preferredBlockSize;
+ this.setReplication(replication);
+ this.setPreferredBlockSize(preferredBlockSize);
blocks = blklist;
}
@@ -70,14 +76,31 @@ class INodeFile extends INode {
/**
* Get block replication for the file
- * @return block replication
+ * @return block replication value
*/
public short getReplication() {
- return this.blockReplication;
+ return (short) ((header & HEADERMASK) >> BLOCKBITS);
}
- void setReplication(short replication) {
- this.blockReplication = replication;
+ public void setReplication(short replication) {
+ if(replication <= 0)
+ throw new IllegalArgumentException("Unexpected value for the replication");
+ header = ((long)replication << BLOCKBITS) | (header & ~HEADERMASK);
+ }
+
+ /**
+ * Get preferred block size for the file
+ * @return preferred block size in bytes
+ */
+ public long getPreferredBlockSize() {
+ return header & ~HEADERMASK;
+ }
+
+ public void setPreferredBlockSize(long preferredBlkSize)
+ {
+ if((preferredBlkSize < 0) || (preferredBlkSize > ~HEADERMASK ))
+ throw new IllegalArgumentException("Unexpected value for the block size");
+ header = (header & HEADERMASK) | (preferredBlkSize & ~HEADERMASK);
}
/**
@@ -157,20 +180,12 @@ class INodeFile extends INode {
*/
if (blkArr.length > 0 && blkArr[blkArr.length-1] != null &&
isUnderConstruction()) {
- size += preferredBlockSize - blocks[blocks.length-1].getNumBytes();
+ size += getPreferredBlockSize() - blocks[blocks.length-1].getNumBytes();
}
- return size * blockReplication;
+ return size * getReplication();
}
/**
- * Get the preferred block size of the file.
- * @return the number of bytes
- */
- public long getPreferredBlockSize() {
- return preferredBlockSize;
- }
-
- /**
* Return the penultimate allocated block for this file.
*/
Block getPenultimateBlock() {
@@ -187,7 +202,7 @@ class INodeFile extends INode {
return (INodeFileUnderConstruction)this;
}
return new INodeFileUnderConstruction(name,
- blockReplication, modificationTime, preferredBlockSize,
+ getReplication(), modificationTime, getPreferredBlockSize(),
blocks, getPermissionStatus(),
clientName, clientMachine, clientNode);
}
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1077483&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Fri Mar 4 04:19:39 2011
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+
+import org.junit.Test;
+
+public class TestINodeFile {
+
+ static final short BLOCKBITS = 48;
+ static final long BLKSIZE_MAXVALUE = ~(0xffffL << BLOCKBITS);
+
+ private String userName = "Test";
+ private short replication;
+ private long preferredBlockSize;
+
+ /**
+ * Test for the Replication value. Sets a value and checks if it was set
+ * correct.
+ */
+ @Test
+ public void testReplication () {
+ replication = 3;
+ preferredBlockSize = 128*1024*1024;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null, replication,
+ 0L, 0L, preferredBlockSize);
+ assertEquals("True has to be returned in this case", replication,
+ inf.getReplication());
+ }
+
+ /**
+ * IllegalArgumentException is expected for setting below lower bound
+ * for Replication.
+ * @throws IllegalArgumentException as the result
+ */
+ @Test(expected=IllegalArgumentException.class)
+ public void testReplicationBelowLowerBound ()
+ throws IllegalArgumentException {
+ replication = -1;
+ preferredBlockSize = 128*1024*1024;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null, replication,
+ 0L, 0L, preferredBlockSize);
+ }
+
+ /**
+ * Test for the PreferredBlockSize value. Sets a value and checks if it was
+ * set correct.
+ */
+ @Test
+ public void testPreferredBlockSize () {
+ replication = 3;
+ preferredBlockSize = 128*1024*1024;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null, replication,
+ 0L, 0L, preferredBlockSize);
+ assertEquals("True has to be returned in this case", preferredBlockSize,
+ inf.getPreferredBlockSize());
+ }
+
+ @Test
+ public void testPreferredBlockSizeUpperBound () {
+ replication = 3;
+ preferredBlockSize = BLKSIZE_MAXVALUE;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null, replication,
+ 0L, 0L, preferredBlockSize);
+ assertEquals("True has to be returned in this case", BLKSIZE_MAXVALUE,
+ inf.getPreferredBlockSize());
+ }
+
+ /**
+ * IllegalArgumentException is expected for setting below lower bound
+ * for PreferredBlockSize.
+ * @throws IllegalArgumentException as the result
+ */
+ @Test(expected=IllegalArgumentException.class)
+ public void testPreferredBlockSizeBelowLowerBound ()
+ throws IllegalArgumentException {
+ replication = 3;
+ preferredBlockSize = -1;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null, replication,
+ 0L, 0L, preferredBlockSize);
+ }
+
+ /**
+ * IllegalArgumentException is expected for setting above upper bound
+ * for PreferredBlockSize.
+ * @throws IllegalArgumentException as the result
+ */
+ @Test(expected=IllegalArgumentException.class)
+ public void testPreferredBlockSizeAboveUpperBound ()
+ throws IllegalArgumentException {
+ replication = 3;
+ preferredBlockSize = BLKSIZE_MAXVALUE+1;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null, replication,
+ 0L, 0L, preferredBlockSize);
+ }
+
+}