You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/05/14 00:25:25 UTC
svn commit: r1102913 - in /hadoop/hdfs/trunk: ./
src/test/hdfs/org/apache/hadoop/hdfs/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/test/
src/test/unit/org/apache/hadoop/hdfs/server/namenode/
Author: todd
Date: Fri May 13 22:25:25 2011
New Revision: 1102913
URL: http://svn.apache.org/viewvc?rev=1102913&view=rev
Log:
HDFS-1899. GenericTestUtils.formatNamenode should be moved to DFSTestUtil. Contributed by Ted Yu.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/test/GenericTestUtils.java
hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri May 13 22:25:25 2011
@@ -410,6 +410,9 @@ Trunk (unreleased changes)
HDFS-1917. Separate hdfs jars from common in ivy configuration. (Eric Yang
via szetszwo)
+ HDFS-1899. GenericTestUtils.formatNamenode should be moved to DFSTestUtil
+ (Ted Yu via todd)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java Fri May 13 22:25:25 2011
@@ -56,10 +56,12 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.hdfs.server.namenode.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -95,6 +97,19 @@ public class DFSTestUtil {
this.maxSize = maxSize;
}
+ /**
+ * when formating a namenode - we must provide clusterid.
+ * @param conf
+ * @throws IOException
+ */
+ public static void formatNameNode(Configuration conf) throws IOException {
+ String clusterId = StartupOption.FORMAT.getClusterId();
+ if(clusterId == null || clusterId.isEmpty())
+ StartupOption.FORMAT.setClusterId("testClusterID");
+
+ NameNode.format(conf);
+ }
+
/** class MyFile contains enough information to recreate the contents of
* a single file.
*/
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Fri May 13 22:25:25 2011
@@ -595,7 +595,7 @@ public class MiniDFSCluster {
// Format and clean out DataNode directories
if (format) {
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
}
if (operation == StartupOption.UPGRADE){
operation.setClusterId(clusterId);
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestHDFSServerPorts.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestHDFSServerPorts.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestHDFSServerPorts.java Fri May 13 22:25:25 2011
@@ -104,7 +104,7 @@ public class TestHDFSServerPorts extends
NameNode.setServiceAddress(config, THIS_HOST);
}
config.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, THIS_HOST);
- GenericTestUtils.formatNamenode(config);
+ DFSTestUtil.formatNameNode(config);
String[] args = new String[] {};
// NameNode will modify config with the ports it bound to
@@ -262,7 +262,7 @@ public class TestHDFSServerPorts extends
Configuration conf2 = new HdfsConfiguration(config);
conf2.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
fileAsURI(new File(hdfsDir, "name2")).toString());
- GenericTestUtils.formatNamenode(conf2);
+ DFSTestUtil.formatNameNode(conf2);
boolean started = canStartNameNode(conf2);
assertFalse(started); // should fail
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java Fri May 13 22:25:25 2011
@@ -105,7 +105,7 @@ public class UpgradeUtilities {
createEmptyDirs(new String[] {datanodeStorage.toString()});
// format and start NameNode and start DataNode
- GenericTestUtils.formatNamenode(config);
+ DFSTestUtil.formatNameNode(config);
cluster = new MiniDFSCluster.Builder(config)
.numDataNodes(1)
.startupOption(StartupOption.REGULAR)
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java Fri May 13 22:25:25 2011
@@ -18,7 +18,8 @@
package org.apache.hadoop.hdfs.server.namenode;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
@@ -27,7 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.DU;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.junit.Test;
public class TestEditLogFileOutputStream {
@@ -37,7 +38,7 @@ public class TestEditLogFileOutputStream
Configuration conf = new Configuration();
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
conf.set("dfs.http.address", "127.0.0.1:0");
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
NameNode nn = new NameNode(conf);
File editLog = nn.getFSImage().getEditLog().getFsEditName();
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java Fri May 13 22:25:25 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.*;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
@@ -320,7 +321,7 @@ public class TestEditLogRace {
public void testSaveImageWhileSyncInProgress() throws Exception {
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
final FSNamesystem namesystem = new FSNamesystem(conf);
try {
@@ -410,7 +411,7 @@ public class TestEditLogRace {
public void testSaveRightBeforeSync() throws Exception {
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
final FSNamesystem namesystem = new FSNamesystem(conf);
try {
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java Fri May 13 22:25:25 2011
@@ -22,6 +22,7 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
@@ -36,7 +37,7 @@ public class TestNNThroughputBenchmark {
Configuration conf = new HdfsConfiguration();
FileSystem.setDefaultUri(conf, "hdfs://localhost:" + 0);
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
String[] args = new String[] {"-op", "all"};
NNThroughputBenchmark.runBenchmark(conf, Arrays.asList(args));
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestReplicationPolicy.java Fri May 13 22:25:25 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import junit.framework.TestCase;
@@ -61,7 +62,7 @@ public class TestReplicationPolicy exten
try {
FileSystem.setDefaultUri(CONF, "hdfs://localhost:0");
CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- GenericTestUtils.formatNamenode(CONF);
+ DFSTestUtil.formatNameNode(CONF);
namenode = new NameNode(CONF);
} catch (IOException e) {
e.printStackTrace();
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Fri May 13 22:25:25 2011
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
@@ -99,7 +100,7 @@ public class TestSaveNamespace {
private void saveNamespaceWithInjectedFault(Fault fault) throws IOException {
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
FSNamesystem fsn = new FSNamesystem(conf);
// Replace the FSImage with a spy
@@ -176,7 +177,7 @@ public class TestSaveNamespace {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true);
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
FSNamesystem fsn = new FSNamesystem(conf);
// Replace the FSImage with a spy
@@ -265,7 +266,7 @@ public class TestSaveNamespace {
public void testSaveWhileEditsRolled() throws Exception {
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
FSNamesystem fsn = new FSNamesystem(conf);
// Replace the FSImage with a spy
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStartup.java Fri May 13 22:25:25 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
@@ -372,7 +373,7 @@ public class TestStartup extends TestCas
conf.set("dfs.name.dir", new File(base_dir, "name").getPath());
conf.setBoolean("dfs.permissions", false);
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
// create an uncompressed image
LOG.info("Create an uncompressed fsimage");
@@ -434,7 +435,7 @@ public class TestStartup extends TestCas
conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, true);
}
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
// create an image
LOG.info("Create an fsimage");
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/test/GenericTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/test/GenericTestUtils.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/test/GenericTestUtils.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/test/GenericTestUtils.java Fri May 13 22:25:25 2011
@@ -34,17 +34,4 @@ public abstract class GenericTestUtils {
public static String getMethodName() {
return Thread.currentThread().getStackTrace()[2].getMethodName();
}
-
- /**
- * when formating a namenode - we must provide clusterid.
- * @param conf
- * @throws IOException
- */
- public static void formatNamenode(Configuration conf) throws IOException {
- String clusterId = StartupOption.FORMAT.getClusterId();
- if(clusterId == null || clusterId.isEmpty())
- StartupOption.FORMAT.setClusterId("testClusterID");
-
- NameNode.format(conf);
- }
}
Modified: hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java?rev=1102913&r1=1102912&r2=1102913&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java (original)
+++ hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java Fri May 13 22:25:25 2011
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
@@ -75,7 +76,7 @@ public class TestNNLeaseRecovery {
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- GenericTestUtils.formatNamenode(conf);
+ DFSTestUtil.formatNameNode(conf);
fsn = spy(new FSNamesystem(conf));
}