You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2013/04/29 23:43:15 UTC
svn commit: r1477358 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/resources/
src/test/java/org/apache/hadoop/hdfs/server/n...
Author: atm
Date: Mon Apr 29 21:41:52 2013
New Revision: 1477358
URL: http://svn.apache.org/r1477358
Log:
HDFS-4305. Add a configurable limit on number of blocks per file, and min block size. Contributed by Andrew Wang.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hdfs-site.xml
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1477358&r1=1477357&r2=1477358&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Apr 29 21:41:52 2013
@@ -219,6 +219,9 @@ Release 2.0.5-beta - UNRELEASED
HDFS-4748. MiniJournalCluster#restartJournalNode leaks resources, which
causes sporadic test failures. (Chris Nauroth via suresh)
+ HDFS-4305. Add a configurable limit on number of blocks per file, and min
+ block size. (Andrew Wang via atm)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1477358&r1=1477357&r2=1477358&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Apr 29 21:41:52 2013
@@ -227,6 +227,10 @@ public class DFSConfigKeys extends Commo
public static final int DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT = 0; // no limit
public static final String DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY = "dfs.namenode.fs-limits.max-directory-items";
public static final int DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT = 0; // no limit
+ public static final String DFS_NAMENODE_MIN_BLOCK_SIZE_KEY = "dfs.namenode.fs-limits.min-block-size";
+ public static final long DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT = 1024*1024;
+ public static final String DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY = "dfs.namenode.fs-limits.max-blocks-per-file";
+ public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 1024*1024;
//Following keys have no defaults
public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir";
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1477358&r1=1477357&r2=1477358&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Apr 29 21:41:52 2013
@@ -349,6 +349,9 @@ public class FSNamesystem implements Nam
private final long maxFsObjects; // maximum number of fs objects
+ private final long minBlockSize; // minimum block size
+ private final long maxBlocksPerFile; // maximum # of blocks per file
+
/**
* The global generation stamp for this file system.
*/
@@ -580,6 +583,10 @@ public class FSNamesystem implements Nam
this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY,
DFS_NAMENODE_MAX_OBJECTS_DEFAULT);
+ this.minBlockSize = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
+ this.maxBlocksPerFile = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT);
this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);
this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT);
@@ -1783,6 +1790,11 @@ public class FSNamesystem implements Nam
final HdfsFileStatus stat;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ if (blockSize < minBlockSize) {
+ throw new IOException("Specified block size is less than configured" +
+ " minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
+ + "): " + blockSize + " < " + minBlockSize);
+ }
writeLock();
try {
startFileInternal(pc, src, permissions, holder, clientMachine, flag,
@@ -2191,7 +2203,12 @@ public class FSNamesystem implements Nam
// This is a retry. Just return the last block.
return onRetryBlock[0];
}
-
+ if (pendingFile.getBlocks().length >= maxBlocksPerFile) {
+ throw new IOException("File has reached the limit on maximum number of"
+ + " blocks (" + DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY
+ + "): " + pendingFile.getBlocks().length + " >= "
+ + maxBlocksPerFile);
+ }
blockSize = pendingFile.getPreferredBlockSize();
clientNode = pendingFile.getClientNode();
replication = pendingFile.getBlockReplication();
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1477358&r1=1477357&r2=1477358&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Mon Apr 29 21:41:52 2013
@@ -239,6 +239,23 @@
</property>
<property>
+ <name>dfs.namenode.fs-limits.min-block-size</name>
+ <value>1048576</value>
+ <description>Minimum block size in bytes, enforced by the Namenode at create
+ time. This prevents the accidental creation of files with tiny block
+ sizes (and thus many blocks), which can degrade
+ performance.</description>
+</property>
+
+<property>
+ <name>dfs.namenode.fs-limits.max-blocks-per-file</name>
+ <value>1048576</value>
+ <description>Maximum number of blocks per file, enforced by the Namenode on
+ write. This prevents the creation of extremely large files which can
+ degrade performance.</description>
+</property>
+
+<property>
<name>dfs.namenode.edits.dir</name>
<value>${dfs.namenode.name.dir}</value>
<description>Determines where on the local filesystem the DFS name node
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java?rev=1477358&r1=1477357&r2=1477358&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java Mon Apr 29 21:41:52 2013
@@ -30,7 +30,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
@@ -174,4 +177,59 @@ public class TestFileLimit {
testFileLimit();
simulatedStorage = false;
}
+
+ @Test(timeout=60000)
+ public void testMaxBlocksPerFileLimit() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ // Make a small block size and a low limit
+ final long blockSize = 4096;
+ final long numBlocks = 2;
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY, numBlocks);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = cluster.getFileSystem();
+ HdfsDataOutputStream fout =
+ (HdfsDataOutputStream)fs.create(new Path("/testmaxfilelimit"));
+ try {
+ // Write maximum number of blocks
+ fout.write(new byte[(int)blockSize*(int)numBlocks]);
+ fout.hflush();
+ // Try to write one more block
+ try {
+ fout.write(new byte[1]);
+ fout.hflush();
+ assert false : "Expected IOException after writing too many blocks";
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("File has reached the limit" +
+ " on maximum number of", e);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout=60000)
+ public void testMinBlockSizeLimit() throws Exception {
+ final long blockSize = 4096;
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, blockSize);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = cluster.getFileSystem();
+
+ try {
+ // Try with min block size
+ fs.create(new Path("/testmblock1"), true, 4096, (short)3, blockSize);
+ try {
+ // Try with min block size - 1
+ fs.create(new Path("/testmblock2"), true, 4096, (short)3, blockSize-1);
+ assert false : "Expected IOException after creating a file with small" +
+ " blocks ";
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Specified block size is less",
+ e);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hdfs-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hdfs-site.xml?rev=1477358&r1=1477357&r2=1477358&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hdfs-site.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hdfs-site.xml Mon Apr 29 21:41:52 2013
@@ -25,5 +25,10 @@
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
+ <!-- Disable min block size since most tests use tiny blocks -->
+ <property>
+ <name>dfs.namenode.fs-limits.min-block-size</name>
+ <value>0</value>
+ </property>
</configuration>