You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sh...@apache.org on 2010/09/15 02:35:54 UTC
svn commit: r997158 - in /hadoop/hdfs/branches/branch-0.21: CHANGES.txt
src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Author: shv
Date: Wed Sep 15 00:35:54 2010
New Revision: 997158
URL: http://svn.apache.org/viewvc?rev=997158&view=rev
Log:
HDFS-1363. Merge -r 997156:997157 from trunk to branch 0.21.
Modified:
hadoop/hdfs/branches/branch-0.21/CHANGES.txt
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=997158&r1=997157&r2=997158&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Wed Sep 15 00:35:54 2010
@@ -916,6 +916,8 @@ Release 0.21.0 - 2010-08-13
HDFS-1267. fuse-dfs does not compile. (Devaraj Das via tomwhite)
+ HDFS-1363. Eliminate second synchronized sections in appendFile(). (shv)
+
Release 0.20.3 - Unreleased
IMPROVEMENTS
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=997158&r1=997157&r2=997158&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Wed Sep 15 00:35:54 2010
@@ -349,19 +349,44 @@ public class BlockManager {
}
/**
- * Convert the last block of the file to an under construction block.
+ * Convert the last block of the file to an under construction block.<p>
+ * The block is converted only if the file has blocks and the last one
+ * is a partial block (its size is less than the preferred block size).
+ * The converted block is returned to the client.
+ * The client uses the returned block locations to form the data pipeline
+ * for this block.<br>
+ * The methods returns null if there is no partial block at the end.
+ * The client is supposed to allocate a new block with the next call.
+ *
* @param fileINode file
- * @param targets data-nodes that will form the pipeline for this block
+ * @return the last block locations if the block is partial or null otherwise
*/
- void convertLastBlockToUnderConstruction(
- INodeFileUnderConstruction fileINode,
- DatanodeDescriptor[] targets) throws IOException {
+ LocatedBlock convertLastBlockToUnderConstruction(
+ INodeFileUnderConstruction fileINode) throws IOException {
BlockInfo oldBlock = fileINode.getLastBlock();
- if(oldBlock == null)
- return;
+ if(oldBlock == null ||
+ fileINode.getPreferredBlockSize() == oldBlock.getNumBytes())
+ return null;
+ assert oldBlock == getStoredBlock(oldBlock) :
+ "last block of the file is not in blocksMap";
+
+ DatanodeDescriptor[] targets = getNodes(oldBlock);
+
BlockInfoUnderConstruction ucBlock =
fileINode.setLastBlock(oldBlock, targets);
blocksMap.replaceBlock(ucBlock);
+
+ // Remove block from replication queue.
+ updateNeededReplications(oldBlock, 0, 0);
+
+ // remove this block from the list of pending blocks to be deleted.
+ for (DatanodeDescriptor dd : targets) {
+ String datanodeId = dd.getStorageID();
+ removeFromInvalidates(datanodeId, oldBlock);
+ }
+
+ long fileLength = fileINode.computeContentSummary().getLength();
+ return getBlockLocation(ucBlock, fileLength - ucBlock.getNumBytes());
}
/**
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=997158&r1=997157&r2=997158&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Sep 15 00:35:54 2010
@@ -1134,7 +1134,23 @@ public class FSNamesystem implements FSC
}
}
- private synchronized void startFileInternal(String src,
+ /**
+ * Create new or open an existing file for append.<p>
+ *
+ * In case of opening the file for append, the method returns the last
+ * block of the file if this is a partial block, which can still be used
+ * for writing more data. The client uses the returned block locations
+ * to form the data pipeline for this block.<br>
+ * The method returns null if the last block is full or if this is a
+ * new file. The client then allocates a new block with the next call
+ * using {@link NameNode#addBlock()}.<p>
+ *
+ * For description of parameters and exceptions thrown see
+ * {@link ClientProtocol#create()}
+ *
+ * @return the last block locations if the block is partial or null otherwise
+ */
+ private synchronized LocatedBlock startFileInternal(String src,
PermissionStatus permissions,
String holder,
String clientMachine,
@@ -1252,9 +1268,8 @@ public class FSNamesystem implements FSC
+ src + " on client " + clientMachine);
else {
//append & create a nonexist file equals to overwrite
- this.startFileInternal(src, permissions, holder, clientMachine,
+ return startFileInternal(src, permissions, holder, clientMachine,
EnumSet.of(CreateFlag.OVERWRITE), createParent, replication, blockSize);
- return;
}
} else if (myFile.isDirectory()) {
throw new IOException("failed to append to directory " + src
@@ -1292,6 +1307,19 @@ public class FSNamesystem implements FSC
dir.replaceNode(src, node, cons);
leaseManager.addLease(cons.getClientName(), src);
+ // convert last block to under-construction
+ LocatedBlock lb =
+ blockManager.convertLastBlockToUnderConstruction(cons);
+
+ if (lb != null && isAccessTokenEnabled) {
+ lb.setAccessToken(accessTokenHandler.generateToken(lb.getBlock()
+ .getBlockId(), EnumSet.of(AccessTokenHandler.AccessMode.WRITE)));
+ }
+ /*if (lb != null && isBlockTokenEnabled) {
+ lb.setBlockToken(blockTokenSecretManager.generateToken(lb.getBlock(),
+ EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE)));
+ }*/
+ return lb;
} else {
// Now we can add the name to the filesystem. This file has no
// blocks associated with it.
@@ -1317,6 +1345,7 @@ public class FSNamesystem implements FSC
+ie.getMessage());
throw ie;
}
+ return null;
}
/**
@@ -1328,52 +1357,12 @@ public class FSNamesystem implements FSC
throw new IOException("Append to hdfs not supported." +
" Please refer to dfs.support.append configuration parameter.");
}
- startFileInternal(src, null, holder, clientMachine, EnumSet.of(CreateFlag.APPEND),
- false, (short)blockManager.maxReplication, (long)0);
+ LocatedBlock lb =
+ startFileInternal(src, null, holder, clientMachine,
+ EnumSet.of(CreateFlag.APPEND),
+ false, (short)blockManager.maxReplication, (long)0);
getEditLog().logSync();
- //
- // Create a LocatedBlock object for the last block of the file
- // to be returned to the client. Return null if the file does not
- // have a partial block at the end.
- //
- LocatedBlock lb = null;
- synchronized (this) {
- INodeFileUnderConstruction file = (INodeFileUnderConstruction)dir.getFileINode(src);
- BlockInfo lastBlock = file.getLastBlock();
- if (lastBlock != null) {
- assert lastBlock == blockManager.getStoredBlock(lastBlock) :
- "last block of the file is not in blocksMap";
- if (file.getPreferredBlockSize() > lastBlock.getNumBytes()) {
- long fileLength = file.computeContentSummary().getLength();
- DatanodeDescriptor[] targets = blockManager.getNodes(lastBlock);
- // remove the replica locations of this block from the node
- for (int i = 0; i < targets.length; i++) {
- targets[i].removeBlock(lastBlock);
- }
- // convert last block to under-construction and set its locations
- blockManager.convertLastBlockToUnderConstruction(file, targets);
-
- lb = new LocatedBlock(lastBlock, targets,
- fileLength-lastBlock.getNumBytes());
- if (isAccessTokenEnabled) {
- lb.setAccessToken(accessTokenHandler.generateToken(lb.getBlock()
- .getBlockId(), EnumSet.of(AccessTokenHandler.AccessMode.WRITE)));
- }
-
- // Remove block from replication queue.
- blockManager.updateNeededReplications(lastBlock, 0, 0);
-
- // remove this block from the list of pending blocks to be deleted.
- // This reduces the possibility of triggering HADOOP-1349.
- //
- for (DatanodeDescriptor dd : targets) {
- String datanodeId = dd.getStorageID();
- blockManager.removeFromInvalidates(datanodeId, lastBlock);
- }
- }
- }
- }
if (lb != null) {
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: file "