You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2012/02/08 06:38:29 UTC
svn commit: r1241780 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
Author: atm
Date: Wed Feb 8 05:38:29 2012
New Revision: 1241780
URL: http://svn.apache.org/viewvc?rev=1241780&view=rev
Log:
HDFS-2764. TestBackupNode is racy. Contributed by Aaron T. Myers.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1241780&r1=1241779&r2=1241780&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Feb 8 05:38:29 2012
@@ -197,6 +197,8 @@ Trunk (unreleased changes)
HDFS-2759. Pre-allocate HDFS edit log files after writing version number.
(atm)
+ HDFS-2764. TestBackupNode is racy. (atm)
+
Release 0.23.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1241780&r1=1241779&r2=1241780&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Wed Feb 8 05:38:29 2012
@@ -34,6 +34,7 @@ import java.util.Properties;
import java.util.Set;
import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
@@ -62,6 +63,8 @@ import static org.mockito.Mockito.mock;
*/
public abstract class FSImageTestUtil {
+ public static final Log LOG = LogFactory.getLog(FSImageTestUtil.class);
+
/**
* The position in the fsimage header where the txid is
* written.
@@ -379,6 +382,8 @@ public abstract class FSImageTestUtil {
List<Integer> txids) {
for (File nameDir : getNameNodeCurrentDirs(cluster)) {
+ LOG.info("examining name dir with files: " +
+ Joiner.on(",").join(nameDir.listFiles()));
// Should have fsimage_N for the three checkpoints
for (long checkpointTxId : txids) {
File image = new File(nameDir,
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1241780&r1=1241779&r2=1241780&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Wed Feb 8 05:38:29 2012
@@ -17,14 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.junit.Assert.*;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.List;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -41,13 +41,15 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
+import org.junit.Before;
+import org.junit.Test;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
-public class TestBackupNode extends TestCase {
+public class TestBackupNode {
public static final Log LOG = LogFactory.getLog(TestBackupNode.class);
@@ -58,8 +60,8 @@ public class TestBackupNode extends Test
static final String BASE_DIR = MiniDFSCluster.getBaseDirectory();
- protected void setUp() throws Exception {
- super.setUp();
+ @Before
+ public void setUp() throws Exception {
File baseDir = new File(BASE_DIR);
if(baseDir.exists())
if(!(FileUtil.fullyDelete(baseDir)))
@@ -90,8 +92,7 @@ public class TestBackupNode extends Test
return (BackupNode)NameNode.createNameNode(new String[]{startupOpt.getName()}, c);
}
- void waitCheckpointDone(
- MiniDFSCluster cluster, BackupNode backup, long txid) {
+ void waitCheckpointDone(MiniDFSCluster cluster, long txid) {
long thisCheckpointTxId;
do {
try {
@@ -99,16 +100,16 @@ public class TestBackupNode extends Test
"checkpoint txid should increase above " + txid);
Thread.sleep(1000);
} catch (Exception e) {}
- thisCheckpointTxId = backup.getFSImage().getStorage()
+ // The checkpoint is not done until the nn has received it from the bn
+ thisCheckpointTxId = cluster.getNameNode().getFSImage().getStorage()
.getMostRecentCheckpointTxId();
-
} while (thisCheckpointTxId < txid);
-
// Check that the checkpoint got uploaded to NN successfully
FSImageTestUtil.assertNNHasCheckpoints(cluster,
Collections.singletonList((int)thisCheckpointTxId));
}
+ @Test
public void testCheckpointNode() throws Exception {
testCheckpoint(StartupOption.CHECKPOINT);
}
@@ -118,6 +119,7 @@ public class TestBackupNode extends Test
* and keep in sync, even while the NN rolls, checkpoints
* occur, etc.
*/
+ @Test
public void testBackupNodeTailsEdits() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
@@ -235,6 +237,7 @@ public class TestBackupNode extends Test
FSImageTestUtil.assertParallelFilesAreIdentical(dirs, ImmutableSet.of("VERSION"));
}
+ @Test
public void testBackupNode() throws Exception {
testCheckpoint(StartupOption.BACKUP);
}
@@ -274,7 +277,7 @@ public class TestBackupNode extends Test
//
long txid = cluster.getNameNodeRpc().getTransactionID();
backup = startBackupNode(conf, op, 1);
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
assertTrue(e.getLocalizedMessage(), false);
@@ -309,7 +312,7 @@ public class TestBackupNode extends Test
//
backup = startBackupNode(conf, op, 1);
long txid = cluster.getNameNodeRpc().getTransactionID();
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
for (int i = 0; i < 10; i++) {
fileSys.mkdirs(new Path("file_" + i));
@@ -317,11 +320,11 @@ public class TestBackupNode extends Test
txid = cluster.getNameNodeRpc().getTransactionID();
backup.doCheckpoint();
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
txid = cluster.getNameNodeRpc().getTransactionID();
backup.doCheckpoint();
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
// Try BackupNode operations
InetSocketAddress add = backup.getNameNodeAddress();