You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2012/08/29 03:22:38 UTC
svn commit: r1378414 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
src/test/java/org/apache/hadoop/hdfs/TestModTime.java
Author: atm
Date: Wed Aug 29 01:22:38 2012
New Revision: 1378414
URL: http://svn.apache.org/viewvc?rev=1378414&view=rev
Log:
HDFS-3864. NN does not update internal file mtime for OP_CLOSE when reading from the edit log. Contributed by Aaron T. Myers.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1378414&r1=1378413&r2=1378414&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Aug 29 01:22:38 2012
@@ -522,6 +522,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3849. When re-loading the FSImage, we should clear the existing
genStamp and leases. (Colin Patrick McCabe via atm)
+ HDFS-3864. NN does not update internal file mtime for OP_CLOSE when reading
+ from the edit log. (atm)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1378414&r1=1378413&r2=1378414&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Wed Aug 29 01:22:38 2012
@@ -304,7 +304,9 @@ public class FSEditLogLoader {
addCloseOp.path);
}
- // Update in-memory data structures
+ // Update the salient file attributes.
+ oldFile.setAccessTime(addCloseOp.atime);
+ oldFile.setModificationTimeForce(addCloseOp.mtime);
updateBlocks(fsDir, addCloseOp, oldFile);
// Now close the file
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java?rev=1378414&r1=1378413&r2=1378414&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java Wed Aug 29 01:22:38 2012
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.util.Random;
@@ -32,12 +33,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.util.ThreadUtil;
import org.junit.Test;
/**
* This class tests the decommissioning of nodes.
*/
public class TestModTime {
+
static final long seed = 0xDEADBEEFL;
static final int blockSize = 8192;
static final int fileSize = 16384;
@@ -186,6 +189,46 @@ public class TestModTime {
cluster.shutdown();
}
}
+
+ /**
+ * Regression test for HDFS-3864 - NN does not update internal file mtime for
+ * OP_CLOSE when reading from the edit log.
+ */
+ @Test
+ public void testModTimePersistsAfterRestart() throws IOException {
+ final long sleepTime = 10; // 10 milliseconds
+ MiniDFSCluster cluster = null;
+ FileSystem fs = null;
+ Configuration conf = new HdfsConfiguration();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ fs = cluster.getFileSystem();
+ Path testPath = new Path("/test");
+
+ // Open a file, and get its initial modification time.
+ OutputStream out = fs.create(testPath);
+ long initialModTime = fs.getFileStatus(testPath).getModificationTime();
+ assertTrue(initialModTime > 0);
+
+ // Wait and then close the file. Ensure that the mod time goes up.
+ ThreadUtil.sleepAtLeastIgnoreInterrupts(sleepTime);
+ out.close();
+ long modTimeAfterClose = fs.getFileStatus(testPath).getModificationTime();
+ assertTrue(modTimeAfterClose >= initialModTime + sleepTime);
+
+ // Restart the NN, and make sure that the later mod time is still used.
+ cluster.restartNameNode();
+ long modTimeAfterRestart = fs.getFileStatus(testPath).getModificationTime();
+ assertEquals(modTimeAfterClose, modTimeAfterRestart);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
public static void main(String[] args) throws Exception {
new TestModTime().testModTime();