You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2013/05/01 01:02:38 UTC
svn commit: r1477868 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/
hadoop-hdfs-httpfs/src/main/resources...
Author: szetszwo
Date: Tue Apr 30 23:02:35 2013
New Revision: 1477868
URL: http://svn.apache.org/r1477868
Log:
Merge r1476453 through r1477867 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSCustomUserName.java
- copied unchanged from r1477867, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSCustomUserName.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestValidateConfigurationSettings.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hdfs-site.xml
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1476453-1477867
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java Tue Apr 30 23:02:35 2013
@@ -193,7 +193,7 @@ public class HttpFSParametersProvider ex
* Constructor.
*/
public DoAsParam() {
- super(NAME, null, UserProvider.USER_PATTERN);
+ super(NAME, null, UserProvider.getUserPattern());
}
/**
@@ -248,7 +248,7 @@ public class HttpFSParametersProvider ex
* Constructor.
*/
public GroupParam() {
- super(NAME, null, UserProvider.USER_PATTERN);
+ super(NAME, null, UserProvider.getUserPattern());
}
}
@@ -344,7 +344,7 @@ public class HttpFSParametersProvider ex
* Constructor.
*/
public OwnerParam() {
- super(NAME, null, UserProvider.USER_PATTERN);
+ super(NAME, null, UserProvider.getUserPattern());
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java Tue Apr 30 23:02:35 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.lib.server.ServerException;
import org.apache.hadoop.lib.service.FileSystemAccess;
import org.apache.hadoop.lib.servlet.ServerWebApp;
+import org.apache.hadoop.lib.wsrs.UserProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,6 +103,9 @@ public class HttpFSServerWebApp extends
LOG.info("Connects to Namenode [{}]",
get().get(FileSystemAccess.class).getFileSystemConfiguration().
get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY));
+ String userPattern = getConfig().get(UserProvider.USER_PATTERN_KEY,
+ UserProvider.USER_PATTERN_DEFAULT);
+ UserProvider.setUserPattern(userPattern);
}
/**
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java Tue Apr 30 23:02:35 2013
@@ -41,12 +41,27 @@ public class UserProvider extends Abstra
public static final String USER_NAME_PARAM = "user.name";
- public static final Pattern USER_PATTERN = Pattern.compile("^[A-Za-z_][A-Za-z0-9._-]*[$]?$");
+
+ public static final String USER_PATTERN_KEY
+ = "httpfs.user.provider.user.pattern";
+
+ public static final String USER_PATTERN_DEFAULT
+ = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
+
+ private static Pattern userPattern = Pattern.compile(USER_PATTERN_DEFAULT);
+
+ public static void setUserPattern(String pattern) {
+ userPattern = Pattern.compile(pattern);
+ }
+
+ public static Pattern getUserPattern() {
+ return userPattern;
+ }
static class UserParam extends StringParam {
public UserParam(String user) {
- super(USER_NAME_PARAM, user, USER_PATTERN);
+ super(USER_NAME_PARAM, user, getUserPattern());
}
@Override
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml Tue Apr 30 23:02:35 2013
@@ -226,4 +226,12 @@
</description>
</property>
+ <property>
+ <name>httpfs.user.provider.user.pattern</name>
+ <value>^[A-Za-z_][A-Za-z0-9._-]*[$]?$</value>
+ <description>
+ Valid pattern for user and group names, it must be a valid java regex.
+ </description>
+ </property>
+
</configuration>
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java Tue Apr 30 23:02:35 2013
@@ -104,34 +104,39 @@ public class TestUserProvider {
@Test
@TestException(exception = IllegalArgumentException.class)
public void userNameEmpty() {
- UserProvider.UserParam userParam = new UserProvider.UserParam("username");
- userParam.parseParam("");
+ new UserProvider.UserParam("");
}
@Test
@TestException(exception = IllegalArgumentException.class)
public void userNameInvalidStart() {
- UserProvider.UserParam userParam = new UserProvider.UserParam("username");
- userParam.parseParam("1x");
+ new UserProvider.UserParam("1x");
}
@Test
@TestException(exception = IllegalArgumentException.class)
public void userNameInvalidDollarSign() {
- UserProvider.UserParam userParam = new UserProvider.UserParam("username");
- userParam.parseParam("1$x");
+ new UserProvider.UserParam("1$x");
}
@Test
public void userNameMinLength() {
- UserProvider.UserParam userParam = new UserProvider.UserParam("username");
- assertNotNull(userParam.parseParam("a"));
+ new UserProvider.UserParam("a");
}
@Test
public void userNameValidDollarSign() {
- UserProvider.UserParam userParam = new UserProvider.UserParam("username");
- assertNotNull(userParam.parseParam("a$"));
+ new UserProvider.UserParam("a$");
+ }
+
+ @Test
+ public void customUserPattern() {
+ try {
+ UserProvider.setUserPattern("1");
+ new UserProvider.UserParam("1");
+ } finally {
+ UserProvider.setUserPattern(UserProvider.USER_PATTERN_DEFAULT);
+ }
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Apr 30 23:02:35 2013
@@ -22,9 +22,6 @@ Trunk (Unreleased)
Azure environments. (See breakdown of tasks below for subtasks and
contributors)
- HDFS-2576. Enhances the DistributedFileSystem's create API so that clients
- can specify favored datanodes for a file's blocks. (ddas)
-
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
@@ -268,7 +265,10 @@ Trunk (Unreleased)
HDFS-4761. When resetting FSDirectory, the inodeMap should also be reset.
(Jing Zhao via szetszwo)
- BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
+ HDFS-4687. TestDelegationTokenForProxyUser#testWebHdfsDoAs is flaky with
+ JDK7. (Andrew Wang via atm)
+
+ BREAKDOWN OF HADOOP-8562 and HDFS-3602 SUBTASKS AND RELATED JIRAS
HDFS-4145. Merge hdfs cmd line scripts from branch-1-win. (David Lao,
Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh)
@@ -311,6 +311,27 @@ Trunk (Unreleased)
HDFS-4584. Skip TestNNWithQJM.testNewNamenodeTakesOverWriter() on Windows.
(Arpit Agarwal via szetszwo)
+ HDFS-4741. TestStorageRestore#testStorageRestoreFailure fails on Windows.
+ (Arpit Agarwal via suresh)
+
+ HDFS-4743. TestNNStorageRetentionManager fails on Windows.
+ (Chris Nauroth via suresh)
+
+ HDFS-4740. Fixes for a few test failures on Windows.
+ (Arpit Agarwal via suresh)
+
+ HDFS-4722. TestGetConf#testFederation times out on Windows.
+ (Ivan Mitic via suresh)
+
+ HDFS-4705. Address HDFS test failures on Windows because of invalid
+ dfs.namenode.name.dir. (Ivan Mitic via suresh)
+
+ HDFS-4734. HDFS Tests that use ShellCommandFencer are broken on Windows.
+ (Arpit Agarwal via suresh)
+
+ HDFS-4610. Use common utils FileUtil#setReadable/Writable/Executable and
+ FileUtil#canRead/Write/Execute. (Ivan Mitic via suresh)
+
BREAKDOWN OF HDFS-347 SUBTASKS AND RELATED JIRAS
HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes.
@@ -373,6 +394,9 @@ Release 2.0.5-beta - UNRELEASED
HDFS-4434. Provide a mapping from INodeId to INode. (suresh)
+ HDFS-4305. Add a configurable limit on number of blocks per file, and min
+ block size. (Andrew Wang via atm)
+
NEW FEATURES
HDFS-1804. Add a new block-volume device choosing policy that looks at
@@ -391,6 +415,9 @@ Release 2.0.5-beta - UNRELEASED
HDFS-4502. JsonUtil.toFileStatus(..) should check if the fileId property
exists. (Brandon Li via suresh)
+ HDFS-2576. Enhances the DistributedFileSystem's create API so that clients
+ can specify favored datanodes for a file's blocks. (ddas)
+
IMPROVEMENTS
HDFS-4222. NN is unresponsive and loses heartbeats from DNs when
@@ -582,6 +609,17 @@ Release 2.0.5-beta - UNRELEASED
HDFS-4745. TestDataTransferKeepalive#testSlowReader has race condition that
causes sporadic failure. (Chris Nauroth via suresh)
+ HDFS-4768. File handle leak in datanode when a block pool is removed.
+ (Chris Nauroth via suresh)
+
+ HDFS-4748. MiniJournalCluster#restartJournalNode leaks resources, which
+ causes sporadic test failures. (Chris Nauroth via suresh)
+
+ HDFS-4733. Make HttpFS username pattern configurable. (tucu via atm)
+
+ HDFS-4778. Fixes some issues that the first patch on HDFS-2576 missed.
+ (ddas)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml Tue Apr 30 23:02:35 2013
@@ -33,6 +33,7 @@ http://maven.apache.org/xsd/maven-4.0.0.
<properties>
<hadoop.component>hdfs</hadoop.component>
+ <hadoop.common.build.dir>${basedir}/../../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
</properties>
<dependencies>
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1476453-1477867
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Apr 30 23:02:35 2013
@@ -227,6 +227,10 @@ public class DFSConfigKeys extends Commo
public static final int DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT = 0; // no limit
public static final String DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY = "dfs.namenode.fs-limits.max-directory-items";
public static final int DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT = 0; // no limit
+ public static final String DFS_NAMENODE_MIN_BLOCK_SIZE_KEY = "dfs.namenode.fs-limits.min-block-size";
+ public static final long DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT = 1024*1024;
+ public static final String DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY = "dfs.namenode.fs-limits.max-blocks-per-file";
+ public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 1024*1024;
//Following keys have no defaults
public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir";
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Tue Apr 30 23:02:35 2013
@@ -170,7 +170,8 @@ public class BlockPlacementPolicyDefault
results.add(remainingTargets[i]);
}
}
- return results.toArray(new DatanodeDescriptor[results.size()]);
+ return getPipeline(writer,
+ results.toArray(new DatanodeDescriptor[results.size()]));
} catch (NotEnoughReplicasException nr) {
// Fall back to regular block placement disregarding favored nodes hint
return chooseTarget(src, numOfReplicas, writer,
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Tue Apr 30 23:02:35 2013
@@ -339,7 +339,6 @@ public class DatanodeManager {
*
* @param address hostaddress:transfer address
* @return the best match for the given datanode
- * @throws IOException when no datanode is found for given address
*/
DatanodeDescriptor getDatanodeDescriptor(String address) {
DatanodeDescriptor node = null;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java Tue Apr 30 23:02:35 2013
@@ -448,7 +448,7 @@ public abstract class Storage extends St
LOG.warn(rootPath + "is not a directory");
return StorageState.NON_EXISTENT;
}
- if (!root.canWrite()) {
+ if (!FileUtil.canWrite(root)) {
LOG.warn("Cannot access storage directory " + rootPath);
return StorageState.NON_EXISTENT;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java Tue Apr 30 23:02:35 2013
@@ -267,7 +267,10 @@ public class DataBlockScanner implements
}
public synchronized void removeBlockPool(String blockPoolId) {
- blockPoolScannerMap.remove(blockPoolId);
+ BlockPoolSliceScanner bpss = blockPoolScannerMap.remove(blockPoolId);
+ if (bpss != null) {
+ bpss.shutdown();
+ }
LOG.info("Removed bpid="+blockPoolId+" from blockPoolScannerMap");
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java Tue Apr 30 23:02:35 2013
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -128,7 +129,7 @@ class FSImagePreTransactionalStorageInsp
static long readCheckpointTime(StorageDirectory sd) throws IOException {
File timeFile = NNStorage.getStorageFile(sd, NameNodeFile.TIME);
long timeStamp = 0L;
- if (timeFile.exists() && timeFile.canRead()) {
+ if (timeFile.exists() && FileUtil.canRead(timeFile)) {
DataInputStream in = new DataInputStream(new FileInputStream(timeFile));
try {
timeStamp = in.readLong();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Apr 30 23:02:35 2013
@@ -373,6 +373,9 @@ public class FSNamesystem implements Nam
private final long maxFsObjects; // maximum number of fs objects
+ private final long minBlockSize; // minimum block size
+ private final long maxBlocksPerFile; // maximum # of blocks per file
+
/**
* The global generation stamp for this file system.
*/
@@ -604,6 +607,10 @@ public class FSNamesystem implements Nam
this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY,
DFS_NAMENODE_MAX_OBJECTS_DEFAULT);
+ this.minBlockSize = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
+ this.maxBlocksPerFile = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT);
this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);
this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT);
@@ -1838,6 +1845,11 @@ public class FSNamesystem implements Nam
final HdfsFileStatus stat;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ if (blockSize < minBlockSize) {
+ throw new IOException("Specified block size is less than configured" +
+ " minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
+ + "): " + blockSize + " < " + minBlockSize);
+ }
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
try {
@@ -2260,7 +2272,12 @@ public class FSNamesystem implements Nam
// This is a retry. Just return the last block.
return onRetryBlock[0];
}
-
+ if (pendingFile.getBlocks().length >= maxBlocksPerFile) {
+ throw new IOException("File has reached the limit on maximum number of"
+ + " blocks (" + DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY
+ + "): " + pendingFile.getBlocks().length + " >= "
+ + maxBlocksPerFile);
+ }
blockSize = pendingFile.getPreferredBlockSize();
clientNode = pendingFile.getClientNode();
replication = pendingFile.getFileReplication();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Apr 30 23:02:35 2013
@@ -34,6 +34,7 @@ import java.util.concurrent.CopyOnWriteA
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
@@ -230,8 +231,8 @@ public class NNStorage extends Storage i
File root = sd.getRoot();
LOG.info("currently disabled dir " + root.getAbsolutePath() +
"; type="+sd.getStorageDirType()
- + ";canwrite="+root.canWrite());
- if(root.exists() && root.canWrite()) {
+ + ";canwrite="+FileUtil.canWrite(root));
+ if(root.exists() && FileUtil.canWrite(root)) {
LOG.info("restoring dir " + sd.getRoot().getAbsolutePath());
this.addStorageDir(sd); // restore
this.removedStorageDirs.remove(sd);
@@ -505,7 +506,7 @@ public class NNStorage extends Storage i
dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
sd = it.next();
File fsImage = getStorageFile(sd, NameNodeFile.IMAGE, txid);
- if(sd.getRoot().canRead() && fsImage.exists())
+ if(FileUtil.canRead(sd.getRoot()) && fsImage.exists())
return fsImage;
}
return null;
@@ -722,7 +723,7 @@ public class NNStorage extends Storage i
private File findFile(NameNodeDirType dirType, String name) {
for (StorageDirectory sd : dirIterable(dirType)) {
File candidate = new File(sd.getCurrentDir(), name);
- if (sd.getCurrentDir().canRead() &&
+ if (FileUtil.canRead(sd.getCurrentDir()) &&
candidate.exists()) {
return candidate;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java Tue Apr 30 23:02:35 2013
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.Time;
@@ -102,7 +103,7 @@ public class TransferFsImage {
assert !dstFiles.isEmpty() : "No checkpoint targets.";
for (File f : dstFiles) {
- if (f.exists() && f.canRead()) {
+ if (f.exists() && FileUtil.canRead(f)) {
LOG.info("Skipping download of remote edit log " +
log + " since it already is stored locally at " + f);
return;
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1476453-1477867
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Tue Apr 30 23:02:35 2013
@@ -239,6 +239,23 @@
</property>
<property>
+ <name>dfs.namenode.fs-limits.min-block-size</name>
+ <value>1048576</value>
+ <description>Minimum block size in bytes, enforced by the Namenode at create
+ time. This prevents the accidental creation of files with tiny block
+ sizes (and thus many blocks), which can degrade
+ performance.</description>
+</property>
+
+<property>
+ <name>dfs.namenode.fs-limits.max-blocks-per-file</name>
+ <value>1048576</value>
+ <description>Maximum number of blocks per file, enforced by the Namenode on
+ write. This prevents the creation of extremely large files which can
+ degrade performance.</description>
+</property>
+
+<property>
<name>dfs.namenode.edits.dir</name>
<value>${dfs.namenode.name.dir}</value>
<description>Determines where on the local filesystem the DFS name node
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1476453-1477867
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1476453-1477867
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1476453-1477867
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1476453-1477867
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Tue Apr 30 23:02:35 2013
@@ -681,9 +681,9 @@ public class MiniDFSCluster {
sb.append("\tabsolute:").append(path.getAbsolutePath()).append("\n");
sb.append("\tpermissions: ");
sb.append(path.isDirectory() ? "d": "-");
- sb.append(path.canRead() ? "r" : "-");
- sb.append(path.canWrite() ? "w" : "-");
- sb.append(path.canExecute() ? "x" : "-");
+ sb.append(FileUtil.canRead(path) ? "r" : "-");
+ sb.append(FileUtil.canWrite(path) ? "w" : "-");
+ sb.append(FileUtil.canExecute(path) ? "x" : "-");
sb.append("\n");
path = path.getParentFile();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Tue Apr 30 23:02:35 2013
@@ -29,11 +29,11 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.apache.hadoop.util.Shell;
+
+import static org.junit.Assert.*;
+import org.junit.Assume;
+import static org.hamcrest.CoreMatchers.*;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -619,19 +619,25 @@ public class TestDFSUtil {
assertEquals(1, uris.size());
assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR)));
+ }
+
+ @Test (timeout=15000)
+ public void testLocalhostReverseLookup() {
+ // 127.0.0.1 -> localhost reverse resolution does not happen on Windows.
+ Assume.assumeTrue(!Shell.WINDOWS);
// Make sure when config FS_DEFAULT_NAME_KEY using IP address,
// it will automatically convert it to hostname
- conf = new HdfsConfiguration();
+ HdfsConfiguration conf = new HdfsConfiguration();
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:8020");
- uris = DFSUtil.getNameServiceUris(conf);
+ Collection<URI> uris = DFSUtil.getNameServiceUris(conf);
assertEquals(1, uris.size());
for (URI uri : uris) {
- assertFalse(uri.getHost().equals("127.0.0.1"));
+ assertThat(uri.getHost(), not("127.0.0.1"));
}
}
-
- @Test
+
+ @Test (timeout=15000)
public void testIsValidName() {
assertFalse(DFSUtil.isValidName("/foo/../bar"));
assertFalse(DFSUtil.isValidName("/foo/./bar"));
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java Tue Apr 30 23:02:35 2013
@@ -155,10 +155,12 @@ public class TestLargeBlock {
}
/**
- * Test for block size of 2GB + 512B
+ * Test for block size of 2GB + 512B. This test can take a rather long time to
+ * complete on Windows (reading the file back can be slow) so we use a larger
+ * timeout here.
* @throws IOException in case of errors
*/
- @Test(timeout = 120000)
+ @Test (timeout = 900000)
public void testLargeBlockSize() throws IOException {
final long blockSize = 2L * 1024L * 1024L * 1024L + 512L; // 2GB + 512B
runTest(blockSize);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java Tue Apr 30 23:02:35 2013
@@ -178,9 +178,9 @@ public class MiniJournalCluster {
conf.set(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY, "127.0.0.1:" +
httpAddrs[i].getPort());
- JournalNode jn = new JournalNode();
- jn.setConf(conf);
- jn.start();
+ nodes[i] = new JournalNode();
+ nodes[i].setConf(conf);
+ nodes[i].start();
}
public int getQuorumSize() {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java Tue Apr 30 23:02:35 2013
@@ -60,24 +60,26 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
import org.apache.log4j.Level;
-import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestDelegationTokenForProxyUser {
- private MiniDFSCluster cluster;
- Configuration config;
+ private static MiniDFSCluster cluster;
+ private static Configuration config;
final private static String GROUP1_NAME = "group1";
final private static String GROUP2_NAME = "group2";
final private static String[] GROUP_NAMES = new String[] { GROUP1_NAME,
GROUP2_NAME };
final private static String REAL_USER = "RealUser";
final private static String PROXY_USER = "ProxyUser";
+ private static UserGroupInformation ugi;
+ private static UserGroupInformation proxyUgi;
private static final Log LOG = LogFactory.getLog(TestDoAsEffectiveUser.class);
- private void configureSuperUserIPAddresses(Configuration conf,
+ private static void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
ArrayList<String> ipList = new ArrayList<String>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
@@ -102,8 +104,8 @@ public class TestDelegationTokenForProxy
builder.toString());
}
- @Before
- public void setUp() throws Exception {
+ @BeforeClass
+ public static void setUp() throws Exception {
config = new HdfsConfiguration();
config.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
config.setLong(
@@ -119,21 +121,20 @@ public class TestDelegationTokenForProxy
cluster = new MiniDFSCluster.Builder(config).build();
cluster.waitActive();
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
+ ugi = UserGroupInformation.createRemoteUser(REAL_USER);
+ proxyUgi = UserGroupInformation.createProxyUserForTesting(PROXY_USER, ugi,
+ GROUP_NAMES);
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
if(cluster!=null) {
cluster.shutdown();
}
}
- @Test
+ @Test(timeout=20000)
public void testDelegationTokenWithRealUser() throws IOException {
- UserGroupInformation ugi = UserGroupInformation
- .createRemoteUser(REAL_USER);
- final UserGroupInformation proxyUgi = UserGroupInformation
- .createProxyUserForTesting(PROXY_USER, ugi, GROUP_NAMES);
try {
Token<?>[] tokens = proxyUgi
.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
@@ -154,12 +155,11 @@ public class TestDelegationTokenForProxy
}
}
- @Test
+ @Test(timeout=20000)
public void testWebHdfsDoAs() throws Exception {
WebHdfsTestUtil.LOG.info("START: testWebHdfsDoAs()");
((Log4JLogger)NamenodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger)ExceptionHandler.LOG).getLogger().setLevel(Level.ALL);
- final UserGroupInformation ugi = UserGroupInformation.createRemoteUser(REAL_USER);
WebHdfsTestUtil.LOG.info("ugi.getShortUserName()=" + ugi.getShortUserName());
final WebHdfsFileSystem webhdfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi, config);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Tue Apr 30 23:02:35 2013
@@ -31,6 +31,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.BlockReader;
import org.apache.hadoop.hdfs.BlockReaderFactory;
@@ -91,10 +92,10 @@ public class TestDataNodeVolumeFailure {
@After
public void tearDown() throws Exception {
if(data_fail != null) {
- data_fail.setWritable(true);
+ FileUtil.setWritable(data_fail, true);
}
if(failedDir != null) {
- failedDir.setWritable(true);
+ FileUtil.setWritable(failedDir, true);
}
if(cluster != null) {
cluster.shutdown();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java Tue Apr 30 23:02:35 2013
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -88,8 +89,8 @@ public class TestDataNodeVolumeFailureRe
@After
public void tearDown() throws Exception {
for (int i = 0; i < 3; i++) {
- new File(dataDir, "data"+(2*i+1)).setExecutable(true);
- new File(dataDir, "data"+(2*i+2)).setExecutable(true);
+ FileUtil.setExecutable(new File(dataDir, "data"+(2*i+1)), true);
+ FileUtil.setExecutable(new File(dataDir, "data"+(2*i+2)), true);
}
cluster.shutdown();
}
@@ -131,8 +132,8 @@ public class TestDataNodeVolumeFailureRe
* fail. The client does not retry failed nodes even though
* perhaps they could succeed because just a single volume failed.
*/
- assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false));
- assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false));
/*
* Create file1 and wait for 3 replicas (ie all DNs can still
@@ -168,7 +169,7 @@ public class TestDataNodeVolumeFailureRe
* Now fail a volume on the third datanode. We should be able to get
* three replicas since we've already identified the other failures.
*/
- assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol1, false));
Path file2 = new Path("/test2");
DFSTestUtil.createFile(fs, file2, 1024, (short)3, 1L);
DFSTestUtil.waitReplication(fs, file2, (short)3);
@@ -200,7 +201,7 @@ public class TestDataNodeVolumeFailureRe
* and that it's no longer up. Only wait for two replicas since
* we'll never get a third.
*/
- assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol2, false));
Path file3 = new Path("/test3");
DFSTestUtil.createFile(fs, file3, 1024, (short)3, 1L);
DFSTestUtil.waitReplication(fs, file3, (short)2);
@@ -222,10 +223,10 @@ public class TestDataNodeVolumeFailureRe
* restart, so file creation should be able to succeed after
* restoring the data directories and restarting the datanodes.
*/
- assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true));
- assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true));
- assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true));
- assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, true));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, true));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol1, true));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn3Vol2, true));
cluster.restartDataNodes();
cluster.waitActive();
Path file4 = new Path("/test4");
@@ -261,8 +262,8 @@ public class TestDataNodeVolumeFailureRe
// third healthy so one node in the pipeline will not fail).
File dn1Vol1 = new File(dataDir, "data"+(2*0+1));
File dn2Vol1 = new File(dataDir, "data"+(2*1+1));
- assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false));
- assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false));
Path file1 = new Path("/test1");
DFSTestUtil.createFile(fs, file1, 1024, (short)2, 1L);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java Tue Apr 30 23:02:35 2013
@@ -77,8 +77,8 @@ public class TestDataNodeVolumeFailureTo
@After
public void tearDown() throws Exception {
for (int i = 0; i < 3; i++) {
- new File(dataDir, "data"+(2*i+1)).setExecutable(true);
- new File(dataDir, "data"+(2*i+2)).setExecutable(true);
+ FileUtil.setExecutable(new File(dataDir, "data"+(2*i+1)), true);
+ FileUtil.setExecutable(new File(dataDir, "data"+(2*i+2)), true);
}
cluster.shutdown();
}
@@ -152,7 +152,7 @@ public class TestDataNodeVolumeFailureTo
// Fail a volume on the 2nd DN
File dn2Vol1 = new File(dataDir, "data"+(2*1+1));
- assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false));
// Should only get two replicas (the first DN and the 3rd)
Path file1 = new Path("/test1");
@@ -165,7 +165,7 @@ public class TestDataNodeVolumeFailureTo
// If we restore the volume we should still only be able to get
// two replicas since the DN is still considered dead.
- assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, true));
Path file2 = new Path("/test2");
DFSTestUtil.createFile(fs, file2, 1024, (short)3, 1L);
DFSTestUtil.waitReplication(fs, file2, (short)2);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Tue Apr 30 23:02:35 2013
@@ -27,6 +27,7 @@ import java.net.Socket;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -106,8 +107,8 @@ public class TestDiskError {
}
} finally {
// restore its old permission
- dir1.setWritable(true);
- dir2.setWritable(true);
+ FileUtil.setWritable(dir1, true);
+ FileUtil.setWritable(dir2, true);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java Tue Apr 30 23:02:35 2013
@@ -169,6 +169,8 @@ public class TestAllowFormat {
InetSocketAddress nnAddr2 = new InetSocketAddress(localhost, 9020);
HATestUtil.setFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2);
+ conf.set(DFS_NAMENODE_NAME_DIR_KEY,
+ new File(hdfsDir, "name").getAbsolutePath());
conf.setBoolean(DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, true);
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX, "dummy"),
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Tue Apr 30 23:02:35 2013
@@ -157,7 +157,7 @@ public class TestCheckpoint {
try {
// Simulate the mount going read-only
- dir.setWritable(false);
+ FileUtil.setWritable(dir, false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(false).build();
fail("NN should have failed to start with " + dir + " set unreadable");
@@ -167,7 +167,7 @@ public class TestCheckpoint {
} finally {
cleanup(cluster);
cluster = null;
- dir.setWritable(true);
+ FileUtil.setWritable(dir, true);
}
}
}
@@ -750,9 +750,12 @@ public class TestCheckpoint {
@Test
public void testSeparateEditsDirLocking() throws IOException {
Configuration conf = new HdfsConfiguration();
- File editsDir = new File(MiniDFSCluster.getBaseDirectory() +
- "/testSeparateEditsDirLocking");
-
+ File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
+ File editsDir = new File(MiniDFSCluster.getBaseDirectory(),
+ "testSeparateEditsDirLocking");
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
editsDir.getAbsolutePath());
MiniDFSCluster cluster = null;
@@ -1822,7 +1825,7 @@ public class TestCheckpoint {
StorageDirectory sd1 = storage.getStorageDir(1);
currentDir = sd0.getCurrentDir();
- currentDir.setExecutable(false);
+ FileUtil.setExecutable(currentDir, false);
// Upload checkpoint when NN has a bad storage dir. This should
// succeed and create the checkpoint in the good dir.
@@ -1832,7 +1835,7 @@ public class TestCheckpoint {
new File(sd1.getCurrentDir(), NNStorage.getImageFileName(2)));
// Restore the good dir
- currentDir.setExecutable(true);
+ FileUtil.setExecutable(currentDir, true);
nn.restoreFailedStorage("true");
nn.rollEditLog();
@@ -1843,7 +1846,7 @@ public class TestCheckpoint {
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
} finally {
if (currentDir != null) {
- currentDir.setExecutable(true);
+ FileUtil.setExecutable(currentDir, true);
}
cleanup(secondary);
secondary = null;
@@ -1893,7 +1896,7 @@ public class TestCheckpoint {
StorageDirectory sd0 = storage.getStorageDir(0);
assertEquals(NameNodeDirType.IMAGE, sd0.getStorageDirType());
currentDir = sd0.getCurrentDir();
- currentDir.setExecutable(false);
+ FileUtil.setExecutable(currentDir, false);
// Try to upload checkpoint -- this should fail since there are no
// valid storage dirs
@@ -1906,7 +1909,7 @@ public class TestCheckpoint {
}
// Restore the good dir
- currentDir.setExecutable(true);
+ FileUtil.setExecutable(currentDir, true);
nn.restoreFailedStorage("true");
nn.rollEditLog();
@@ -1917,7 +1920,7 @@ public class TestCheckpoint {
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
} finally {
if (currentDir != null) {
- currentDir.setExecutable(true);
+ FileUtil.setExecutable(currentDir, true);
}
cleanup(secondary);
secondary = null;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Tue Apr 30 23:02:35 2013
@@ -881,14 +881,14 @@ public class TestEditLog {
logDir.mkdirs();
FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
try {
- logDir.setWritable(false);
+ FileUtil.setWritable(logDir, false);
log.openForWrite();
fail("Did no throw exception on only having a bad dir");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
"too few journals successfully started", ioe);
} finally {
- logDir.setWritable(true);
+ FileUtil.setWritable(logDir, true);
log.close();
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Tue Apr 30 23:02:35 2013
@@ -19,20 +19,30 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
import static org.junit.Assert.assertEquals;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
+import org.junit.After;
import org.junit.Test;
public class TestFSNamesystem {
+ @After
+ public void cleanUp() {
+ FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory()));
+ }
+
/**
* Tests that the namenode edits dirs are gotten with duplicates removed
*/
@@ -54,6 +64,9 @@ public class TestFSNamesystem {
@Test
public void testFSNamespaceClearLeases() throws Exception {
Configuration conf = new HdfsConfiguration();
+ File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
+ conf.set(DFS_NAMENODE_NAME_DIR_KEY, nameDir.getAbsolutePath());
+
NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
DFSTestUtil.formatNameNode(conf);
FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java Tue Apr 30 23:02:35 2013
@@ -22,6 +22,7 @@ import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.Random;
+import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -32,6 +33,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -67,7 +69,7 @@ public class TestFavoredNodesEndToEnd {
}
}
- @Test
+ @Test(timeout=180000)
public void testFavoredNodesEndToEnd() throws Exception {
//create 10 files with random preferred nodes
for (int i = 0; i < NUM_FILES; i++) {
@@ -80,11 +82,7 @@ public class TestFavoredNodesEndToEnd {
4096, (short)3, (long)4096, null, datanode);
out.write(SOME_BYTES);
out.close();
- BlockLocation[] locations =
- dfs.getClient().getBlockLocations(p.toUri().getPath(), 0,
- Long.MAX_VALUE);
- //make sure we have exactly one block location, and three hosts
- assertTrue(locations.length == 1 && locations[0].getHosts().length == 3);
+ BlockLocation[] locations = getBlockLocations(p);
//verify the files got created in the right nodes
for (BlockLocation loc : locations) {
String[] hosts = loc.getNames();
@@ -94,7 +92,7 @@ public class TestFavoredNodesEndToEnd {
}
}
- @Test
+ @Test(timeout=180000)
public void testWhenFavoredNodesNotPresent() throws Exception {
//when we ask for favored nodes but the nodes are not there, we should
//get some other nodes. In other words, the write to hdfs should not fail
@@ -110,13 +108,10 @@ public class TestFavoredNodesEndToEnd {
4096, (short)3, (long)4096, null, arbitraryAddrs);
out.write(SOME_BYTES);
out.close();
- BlockLocation[] locations =
- dfs.getClient().getBlockLocations(p.toUri().getPath(), 0,
- Long.MAX_VALUE);
- assertTrue(locations.length == 1 && locations[0].getHosts().length == 3);
+ getBlockLocations(p);
}
- @Test
+ @Test(timeout=180000)
public void testWhenSomeNodesAreNotGood() throws Exception {
//make some datanode not "good" so that even if the client prefers it,
//the namenode would not give it as a replica to write to
@@ -136,12 +131,9 @@ public class TestFavoredNodesEndToEnd {
4096, (short)3, (long)4096, null, addrs);
out.write(SOME_BYTES);
out.close();
- BlockLocation[] locations =
- dfs.getClient().getBlockLocations(p.toUri().getPath(), 0,
- Long.MAX_VALUE);
//reset the state
d.stopDecommission();
- assertTrue(locations.length == 1 && locations[0].getHosts().length == 3);
+ BlockLocation[] locations = getBlockLocations(p);
//also make sure that the datanode[0] is not in the list of hosts
String datanode0 =
datanodes.get(0).getXferAddress().getAddress().getHostAddress()
@@ -153,6 +145,14 @@ public class TestFavoredNodesEndToEnd {
}
}
+ private BlockLocation[] getBlockLocations(Path p) throws Exception {
+ DFSTestUtil.waitReplication(dfs, p, (short)3);
+ BlockLocation[] locations = dfs.getClient().getBlockLocations(
+ p.toUri().getPath(), 0, Long.MAX_VALUE);
+ assertTrue(locations.length == 1 && locations[0].getHosts().length == 3);
+ return locations;
+ }
+
private String[] getStringForInetSocketAddrs(InetSocketAddress[] datanode) {
String strs[] = new String[datanode.length];
for (int i = 0; i < datanode.length; i++) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java Tue Apr 30 23:02:35 2013
@@ -28,7 +28,10 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
@@ -159,4 +162,59 @@ public class TestFileLimit {
testFileLimit();
simulatedStorage = false;
}
+
+ @Test(timeout=60000)
+ public void testMaxBlocksPerFileLimit() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ // Make a small block size and a low limit
+ final long blockSize = 4096;
+ final long numBlocks = 2;
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY, numBlocks);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = cluster.getFileSystem();
+ HdfsDataOutputStream fout =
+ (HdfsDataOutputStream)fs.create(new Path("/testmaxfilelimit"));
+ try {
+ // Write maximum number of blocks
+ fout.write(new byte[(int)blockSize*(int)numBlocks]);
+ fout.hflush();
+ // Try to write one more block
+ try {
+ fout.write(new byte[1]);
+ fout.hflush();
+ assert false : "Expected IOException after writing too many blocks";
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("File has reached the limit" +
+ " on maximum number of", e);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout=60000)
+ public void testMinBlockSizeLimit() throws Exception {
+ final long blockSize = 4096;
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, blockSize);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = cluster.getFileSystem();
+
+ try {
+ // Try with min block size
+ fs.create(new Path("/testmblock1"), true, 4096, (short)3, blockSize);
+ try {
+ // Try with min block size - 1
+ fs.create(new Path("/testmblock2"), true, 4096, (short)3, blockSize-1);
+ assert false : "Expected IOException after creating a file with small" +
+ " blocks ";
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Specified block size is less",
+ e);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java Tue Apr 30 23:02:35 2013
@@ -28,6 +28,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -106,10 +107,10 @@ public class TestNNStorageRetentionFunct
getInProgressEditsFileName(5));
LOG.info("Failing first storage dir by chmodding it");
- sd0.setExecutable(false);
+ FileUtil.setExecutable(sd0, false);
doSaveNamespace(nn);
LOG.info("Restoring accessibility of first storage dir");
- sd0.setExecutable(true);
+ FileUtil.setExecutable(sd0, true);
LOG.info("nothing should have been purged in first storage dir");
assertGlobEquals(cd0, "fsimage_\\d*",
@@ -138,7 +139,7 @@ public class TestNNStorageRetentionFunct
assertGlobEquals(cd0, "edits_.*",
getInProgressEditsFileName(9));
} finally {
- sd0.setExecutable(true);
+ FileUtil.setExecutable(sd0, true);
LOG.info("Shutting down...");
if (cluster != null) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java Tue Apr 30 23:02:35 2013
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.ser
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
+import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
@@ -247,30 +248,32 @@ public class TestNNStorageRetentionManag
.purgeLog(logsPurgedCaptor.capture());
// Check images
- Set<String> purgedPaths = Sets.newHashSet();
+ Set<String> purgedPaths = Sets.newLinkedHashSet();
for (FSImageFile purged : imagesPurgedCaptor.getAllValues()) {
- purgedPaths.add(purged.getFile().toString());
+ purgedPaths.add(fileToPath(purged.getFile()));
}
- Assert.assertEquals(Joiner.on(",").join(tc.expectedPurgedImages),
- Joiner.on(",").join(purgedPaths));
+ Assert.assertEquals(
+ Joiner.on(",").join(filesToPaths(tc.expectedPurgedImages)),
+ Joiner.on(",").join(purgedPaths));
// Check images
purgedPaths.clear();
for (EditLogFile purged : logsPurgedCaptor.getAllValues()) {
- purgedPaths.add(purged.getFile().toString());
+ purgedPaths.add(fileToPath(purged.getFile()));
}
- Assert.assertEquals(Joiner.on(",").join(tc.expectedPurgedLogs),
- Joiner.on(",").join(purgedPaths));
+ Assert.assertEquals(
+ Joiner.on(",").join(filesToPaths(tc.expectedPurgedLogs)),
+ Joiner.on(",").join(purgedPaths));
}
private static class TestCaseDescription {
- private Map<String, FakeRoot> dirRoots = Maps.newHashMap();
- private Set<String> expectedPurgedLogs = Sets.newHashSet();
- private Set<String> expectedPurgedImages = Sets.newHashSet();
+ private Map<File, FakeRoot> dirRoots = Maps.newHashMap();
+ private Set<File> expectedPurgedLogs = Sets.newLinkedHashSet();
+ private Set<File> expectedPurgedImages = Sets.newLinkedHashSet();
private static class FakeRoot {
NameNodeDirType type;
- List<String> files;
+ List<File> files;
FakeRoot(NameNodeDirType type) {
this.type = type;
@@ -280,33 +283,35 @@ public class TestNNStorageRetentionManag
StorageDirectory mockStorageDir() {
return FSImageTestUtil.mockStorageDirectory(
type, false,
- files.toArray(new String[0]));
+ filesToPaths(files).toArray(new String[0]));
}
}
void addRoot(String root, NameNodeDirType dir) {
- dirRoots.put(root, new FakeRoot(dir));
+ dirRoots.put(new File(root), new FakeRoot(dir));
}
- private void addFile(String path) {
- for (Map.Entry<String, FakeRoot> entry : dirRoots.entrySet()) {
- if (path.startsWith(entry.getKey())) {
- entry.getValue().files.add(path);
+ private void addFile(File file) {
+ for (Map.Entry<File, FakeRoot> entry : dirRoots.entrySet()) {
+ if (fileToPath(file).startsWith(fileToPath(entry.getKey()))) {
+ entry.getValue().files.add(file);
}
}
}
void addLog(String path, boolean expectPurge) {
- addFile(path);
+ File file = new File(path);
+ addFile(file);
if (expectPurge) {
- expectedPurgedLogs.add(path);
+ expectedPurgedLogs.add(file);
}
}
void addImage(String path, boolean expectPurge) {
- addFile(path);
+ File file = new File(path);
+ addFile(file);
if (expectPurge) {
- expectedPurgedImages.add(path);
+ expectedPurgedImages.add(file);
}
}
@@ -364,6 +369,30 @@ public class TestNNStorageRetentionManag
}
}
+ /**
+ * Converts a file to a platform-agnostic URI path.
+ *
+ * @param file File to convert
+ * @return String path
+ */
+ private static String fileToPath(File file) {
+ return file.toURI().getPath();
+ }
+
+ /**
+ * Converts multiple files to platform-agnostic URI paths.
+ *
+ * @param files Collection<File> files to convert
+ * @return Collection<String> paths
+ */
+ private static Collection<String> filesToPaths(Collection<File> files) {
+ List<String> paths = Lists.newArrayList();
+ for (File file: files) {
+ paths.add(fileToPath(file));
+ }
+ return paths;
+ }
+
private static NNStorage mockStorageForDirs(final StorageDirectory ... mockDirs)
throws IOException {
NNStorage mockStorage = Mockito.mock(NNStorage.class);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java Tue Apr 30 23:02:35 2013
@@ -17,23 +17,35 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.File;
import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
import org.junit.Test;
public class TestNNThroughputBenchmark {
+ @After
+ public void cleanUp() {
+ FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory()));
+ }
+
/**
* This test runs all benchmarks defined in {@link NNThroughputBenchmark}.
*/
@Test
public void testNNThroughput() throws Exception {
Configuration conf = new HdfsConfiguration();
+ File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
FileSystem.setDefaultUri(conf, "hdfs://localhost:" + 0);
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
DFSTestUtil.formatNameNode(conf);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java Tue Apr 30 23:02:35 2013
@@ -322,12 +322,15 @@ public class TestNameEditsConfigs {
MiniDFSCluster cluster = null;
File nameAndEditsDir = new File(base_dir, "name_and_edits");
File nameAndEditsDir2 = new File(base_dir, "name_and_edits2");
+ File nameDir = new File(base_dir, "name");
// 1
// Bad configuration. Add a directory to dfs.namenode.edits.dir.required
// without adding it to dfs.namenode.edits.dir.
try {
Configuration conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
conf.set(
DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY,
nameAndEditsDir2.toURI().toString());
@@ -353,6 +356,8 @@ public class TestNameEditsConfigs {
// and dfs.namenode.edits.dir.
try {
Configuration conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
conf.setStrings(
DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
nameAndEditsDir.toURI().toString(),
@@ -375,6 +380,8 @@ public class TestNameEditsConfigs {
// dfs.namenode.edits.dir.required.
try {
Configuration conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
conf.setStrings(
DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
nameAndEditsDir.toURI().toString(),