You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by um...@apache.org on 2014/05/12 08:28:26 UTC
svn commit: r1593895 - in
/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: umamahesh
Date: Mon May 12 06:28:26 2014
New Revision: 1593895
URL: http://svn.apache.org/r1593895
Log:
HDFS-6344. Maximum limit on the size of an xattr. Contributed by Yi Liu.
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNConf.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeXAttr.java
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt Mon May 12 06:28:26 2014
@@ -27,6 +27,8 @@ HDFS-2006 (Unreleased)
HDFS-6314. Test cases for XAttrs. (Yi Liu via umamahesh)
+ HDFS-6344. Maximum limit on the size of an xattr. (Yi Liu via umamahesh)
+
OPTIMIZATIONS
HDFS-6346. Optimize OP_SET_XATTRS by persisting single Xattr entry per setXattr/removeXattr api call
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon May 12 06:28:26 2014
@@ -196,6 +196,10 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_NAMENODE_INODE_XATTRS_MAX_LIMIT_KEY = "dfs.namenode.inode.xattrs.max-limit";
public static final int DFS_NAMENODE_INODE_XATTRS_MAX_LIMIT_DEFAULT = 32;
+ public static final String DFS_NAMENODE_XATTR_NAME_MAX_LENGTH_KEY = "dfs.namenode.xattr.name.max-length";
+ public static final int DFS_NAMENODE_XATTR_NAME_MAX_LENGTH_DEFAULT = 64;
+ public static final String DFS_NAMENODE_XATTR_VALUE_MAX_LENGTH_KEY = "dfs.namenode.xattr.value.max-length";
+ public static final int DFS_NAMENODE_XATTR_VALUE_MAX_LENGTH_DEFAULT = 16384;
public static final String DFS_ADMIN = "dfs.cluster.administrators";
public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY = "dfs.https.server.keystore.resource";
public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT = "ssl-server.xml";
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon May 12 06:28:26 2014
@@ -7744,6 +7744,7 @@ public class FSNamesystem implements Nam
private void setXAttrInt(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache) throws IOException {
nnConf.checkXAttrsConfigFlag();
+ checkXAttrSize(xAttr);
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(pc, xAttr);
@@ -7766,6 +7767,22 @@ public class FSNamesystem implements Nam
logAuditEvent(true, "setXAttr", src, null, resultingStat);
}
+ private void checkXAttrSize(XAttr xAttr) {
+ if (xAttr.getName().length() > nnConf.xattrNameMaxLength) {
+ throw new HadoopIllegalArgumentException(
+ "XAttr name is too long, maximum length = "
+ + nnConf.xattrNameMaxLength + ", but now the length = "
+ + xAttr.getName().length());
+ }
+ if (xAttr.getValue() != null
+ && xAttr.getValue().length > nnConf.xattrValueMaxLength) {
+ throw new HadoopIllegalArgumentException(
+ "XAttr value is too long, maximum length = "
+ + nnConf.xattrValueMaxLength + ", but now the length = "
+ + xAttr.getValue().length);
+ }
+ }
+
List<XAttr> getXAttrs(String src, List<XAttr> xAttrs) throws IOException {
nnConf.checkXAttrsConfigFlag();
FSPermissionChecker pc = getPermissionChecker();
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNConf.java?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNConf.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNConf.java Mon May 12 06:28:26 2014
@@ -41,6 +41,9 @@ final class NNConf {
* XAttr-related operations.
*/
private final boolean xattrsEnabled;
+
+ final int xattrNameMaxLength;
+ final int xattrValueMaxLength;
/**
* Creates a new NNConf from configuration.
@@ -51,9 +54,16 @@ final class NNConf {
aclsEnabled = conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
LogFactory.getLog(NNConf.class).info("ACLs enabled? " + aclsEnabled);
- xattrsEnabled = conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
- DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
+ xattrsEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
LogFactory.getLog(NNConf.class).info("XAttrs enabled? " + xattrsEnabled);
+ xattrNameMaxLength = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_XATTR_NAME_MAX_LENGTH_KEY,
+ DFSConfigKeys.DFS_NAMENODE_XATTR_NAME_MAX_LENGTH_DEFAULT);
+ xattrValueMaxLength = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_XATTR_VALUE_MAX_LENGTH_KEY,
+ DFSConfigKeys.DFS_NAMENODE_XATTR_VALUE_MAX_LENGTH_DEFAULT);
}
/**
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java Mon May 12 06:28:26 2014
@@ -195,6 +195,8 @@ public class FSXAttrBaseTest {
* 2. Set xattr with illegal name.
* 3. Set xattr without XAttrSetFlag.
* 4. Set xattr and total number exceeds max limit.
+ * 5. Set xattr and name is too long.
+ * 6. Set xattr and value is too long.
*/
@Test
public void testSetXAttr() throws Exception {
@@ -263,6 +265,21 @@ public class FSXAttrBaseTest {
fs.removeXAttr(path, name1);
fs.removeXAttr(path, name2);
fs.removeXAttr(path, name3);
+
+ // Name length exceeds max limit
+ String longName = "user.abcdefg123456789000";
+ try {
+ fs.setXAttr(path, longName, value1);
+ Assert.fail("Setting xattr should fail if name is too long.");
+ } catch (IOException e) {
+ }
+ // Value length exceeds max limit
+ byte[] longValue = new byte[40];
+ try {
+ fs.setXAttr(path, name1, longValue);
+ Assert.fail("Setting xattr should fail if value is too long.");
+ } catch (IOException e) {
+ }
}
/**
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java Mon May 12 06:28:26 2014
@@ -42,6 +42,8 @@ public class TestFileContextXAttr extend
conf = new Configuration();
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_INODE_XATTRS_MAX_LIMIT_KEY, 3);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_XATTR_NAME_MAX_LENGTH_KEY, 16);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_XATTR_VALUE_MAX_LENGTH_KEY, 32);
initCluster(true);
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeXAttr.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeXAttr.java?rev=1593895&r1=1593894&r2=1593895&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeXAttr.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeXAttr.java Mon May 12 06:28:26 2014
@@ -44,6 +44,8 @@ public class TestNameNodeXAttr extends F
conf = new Configuration();
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_INODE_XATTRS_MAX_LIMIT_KEY, 3);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_XATTR_NAME_MAX_LENGTH_KEY, 16);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_XATTR_VALUE_MAX_LENGTH_KEY, 32);
initCluster(true);
}