You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2010/06/16 23:57:39 UTC
svn commit: r955410 - in /hadoop/common/branches/branch-0.20-append:
CHANGES.txt src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
src/test/org/apache/hadoop/hdfs/TestFileAppend4.java
Author: dhruba
Date: Wed Jun 16 21:57:39 2010
New Revision: 955410
URL: http://svn.apache.org/viewvc?rev=955410&view=rev
Log:
HDFS-1141. completeFile does not check lease ownership.
(Todd Lipcon via dhruba)
Modified:
hadoop/common/branches/branch-0.20-append/CHANGES.txt
hadoop/common/branches/branch-0.20-append/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-0.20-append/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java
Modified: hadoop/common/branches/branch-0.20-append/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/CHANGES.txt?rev=955410&r1=955409&r2=955410&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-append/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-append/CHANGES.txt Wed Jun 16 21:57:39 2010
@@ -31,6 +31,9 @@ Release 0.20-append - Unreleased
HDFS-445. pread should refetch block locations when necessary.
(Todd Lipcon via dhruba)
+ HDFS-1141. completeFile does not check lease ownership.
+ (Todd Lipcon via dhruba)
+
IMPROVEMENTS
BUG FIXES
Modified: hadoop/common/branches/branch-0.20-append/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=955410&r1=955409&r2=955410&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-append/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.20-append/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Jun 16 21:57:39 2010
@@ -1410,24 +1410,19 @@ public class FSNamesystem implements FSC
NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " + src + " for " + holder);
if (isInSafeMode())
throw new SafeModeException("Cannot complete file " + src, safeMode);
- INode iFile = dir.getFileINode(src);
- INodeFileUnderConstruction pendingFile = null;
- Block[] fileBlocks = null;
-
- if (iFile != null && iFile.isUnderConstruction()) {
- pendingFile = (INodeFileUnderConstruction) iFile;
- fileBlocks = dir.getFileBlocks(src);
- }
- if (fileBlocks == null ) {
+
+ INodeFileUnderConstruction pendingFile = checkLease(src, holder);
+ Block[] fileBlocks = dir.getFileBlocks(src);
+
+ if (fileBlocks == null ) {
NameNode.stateChangeLog.warn("DIR* NameSystem.completeFile: "
+ "failed to complete " + src
- + " because dir.getFileBlocks() is null " +
- " and pendingFile is " +
- ((pendingFile == null) ? "null" :
- ("from " + pendingFile.getClientMachine()))
- );
+ + " because dir.getFileBlocks() is null,"
+ + " pending from " + pendingFile.getClientMachine());
return CompleteFileStatus.OPERATION_FAILED;
- } else if (!checkFileProgress(pendingFile, true)) {
+ }
+
+ if (!checkFileProgress(pendingFile, true)) {
return CompleteFileStatus.STILL_WAITING;
}
Modified: hadoop/common/branches/branch-0.20-append/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-append/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java?rev=955410&r1=955409&r2=955410&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-append/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java (original)
+++ hadoop/common/branches/branch-0.20-append/src/test/org/apache/hadoop/hdfs/TestFileAppend4.java Wed Jun 16 21:57:39 2010
@@ -674,6 +674,85 @@ public class TestFileAppend4 extends Tes
}
}
+
+ /**
+ * Test case that stops a writer after finalizing a block but
+ * before calling completeFile, recovers a file from another writer,
+ * starts writing from that writer, and then has the old lease holder
+ * call completeFile
+ */
+ public void testCompleteOtherLeaseHoldersFile() throws Throwable {
+ cluster = new MiniDFSCluster(conf, 3, true, null);
+
+ try {
+ cluster.waitActive();
+ NameNode preSpyNN = cluster.getNameNode();
+ NameNode spyNN = spy(preSpyNN);
+
+ // Delay completeFile
+ DelayAnswer delayer = new DelayAnswer();
+ doAnswer(delayer).when(spyNN).complete(anyString(), anyString());
+
+ DFSClient client = new DFSClient(null, spyNN, conf, null);
+ file1 = new Path("/testRecoverFinalized");
+ final OutputStream stm = client.create("/testRecoverFinalized", true);
+
+ // write 1/2 block
+ AppendTestUtil.write(stm, 0, 4096);
+ final AtomicReference<Throwable> err = new AtomicReference<Throwable>();
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ stm.close();
+ } catch (Throwable t) {
+ err.set(t);
+ }
+ }};
+ t.start();
+ LOG.info("Waiting for close to get to latch...");
+ delayer.waitForCall();
+
+ // At this point, the block is finalized on the DNs, but the file
+ // has not been completed in the NN.
+ // Lose the leases
+ LOG.info("Killing lease checker");
+ client.leasechecker.interruptAndJoin();
+
+ FileSystem fs1 = cluster.getFileSystem();
+ FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(
+ fs1.getConf());
+
+ LOG.info("Recovering file");
+ recoverFile(fs2);
+
+ LOG.info("Opening file for append from new fs");
+ FSDataOutputStream appenderStream = fs2.append(file1);
+
+ LOG.info("Writing some data from new appender");
+ AppendTestUtil.write(appenderStream, 0, 4096);
+
+ LOG.info("Telling old close to proceed.");
+ delayer.proceed();
+ LOG.info("Waiting for close to finish.");
+ t.join();
+ LOG.info("Close finished.");
+
+ // We expect that close will get a "Lease mismatch"
+ // error.
+ Throwable thrownByClose = err.get();
+ assertNotNull(thrownByClose);
+ assertTrue(thrownByClose instanceof IOException);
+ if (!thrownByClose.getMessage().contains(
+ "Lease mismatch"))
+ throw thrownByClose;
+
+ // The appender should be able to close properly
+ appenderStream.close();
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* Test for an intermittent failure of commitBlockSynchronization.
* This could happen if the DN crashed between calling updateBlocks
@@ -710,6 +789,7 @@ public class TestFileAppend4 extends Tes
LOG.info("STOP");
}
+
/**
* Test that when a DN starts up with bbws from a file that got
* removed or finalized when it was down, the block gets deleted.