You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2011/09/09 21:27:21 UTC
svn commit: r1167333 - in /hadoop/common/branches/branch-0.20-security: ./
src/hdfs/org/apache/hadoop/hdfs/server/datanode/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/hdfs/org/apache/hadoop/hdfs/server/protocol/
src/test/org/apache/hadoop/hdf...
Author: jitendra
Date: Fri Sep 9 19:27:21 2011
New Revision: 1167333
URL: http://svn.apache.org/viewvc?rev=1167333&view=rev
Log:
HDFS-1779. After NameNode restart , Clients can not read partial files even after client invokes Sync. Contributed by Uma Maheswara Rao G.
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Fri Sep 9 19:27:21 2011
@@ -107,6 +107,9 @@ Release 0.20.205.0 - unreleased
HADOOP-6722. NetUtils.connect should check that it hasn't connected a socket
to itself. (Todd Lipcon via suresh)
+ HDFS-1779. After NameNode restart , Clients can not read partial files even after
+ client invokes Sync. (Uma Maheswara Rao G via jitendra)
+
IMPROVEMENTS
MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Sep 9 19:27:21 2011
@@ -216,6 +216,7 @@ public class DataNode extends Configured
int socketWriteTimeout = 0;
boolean transferToAllowed = true;
int writePacketSize = 0;
+ private boolean supportAppends;
boolean isBlockTokenEnabled;
BlockTokenSecretManager blockTokenSecretManager;
boolean isBlockTokenInitialized = false;
@@ -264,7 +265,7 @@ public class DataNode extends Configured
DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
datanodeObject = this;
-
+ supportAppends = conf.getBoolean("dfs.support.append", false);
try {
startDataNode(conf, dataDirs, resources);
} catch (IOException ie) {
@@ -673,6 +674,12 @@ public class DataNode extends Configured
dnRegistration.exportedKeys = ExportedBlockKeys.DUMMY_KEYS;
}
+ if (supportAppends) {
+ Block[] bbwReport = data.getBlocksBeingWrittenReport();
+ long[] blocksBeingWritten = BlockListAsLongs
+ .convertToArrayLongs(bbwReport);
+ namenode.blocksBeingWrittenReport(dnRegistration, blocksBeingWritten);
+ }
// random short delay - helps scatter the BR from all DNs
scheduleBlockReport(initialBlockReportDelay);
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Fri Sep 9 19:27:21 2011
@@ -59,6 +59,33 @@ import org.mortbay.log.Log;
*
***************************************************/
public class FSDataset implements FSConstants, FSDatasetInterface {
+
+
+ /** Find the metadata file for the specified block file.
+ * Return the generation stamp from the name of the metafile.
+ */
+ private static long getGenerationStampFromFile(File[] listdir, File blockFile) {
+ String blockName = blockFile.getName();
+ for (int j = 0; j < listdir.length; j++) {
+ String path = listdir[j].getName();
+ if (!path.startsWith(blockName)) {
+ continue;
+ }
+ String[] vals = path.split("_");
+ if (vals.length != 3) { // blk, blkid, genstamp.meta
+ continue;
+ }
+ String[] str = vals[2].split("\\.");
+ if (str.length != 2) {
+ continue;
+ }
+ return Long.parseLong(str[0]);
+ }
+ DataNode.LOG.warn("Block " + blockFile +
+ " does not have a metafile!");
+ return Block.GRANDFATHER_GENERATION_STAMP;
+ }
+
/**
* A data structure than encapsulates a Block along with the full pathname
@@ -183,31 +210,6 @@ public class FSDataset implements FSCons
return children[ lastChildIdx ].addBlock(b, src, true, false);
}
- /** Find the metadata file for the specified block file.
- * Return the generation stamp from the name of the metafile.
- */
- long getGenerationStampFromFile(File[] listdir, File blockFile) {
- String blockName = blockFile.getName();
- for (int j = 0; j < listdir.length; j++) {
- String path = listdir[j].getName();
- if (!path.startsWith(blockName)) {
- continue;
- }
- String[] vals = path.split("_");
- if (vals.length != 3) { // blk, blkid, genstamp.meta
- continue;
- }
- String[] str = vals[2].split("\\.");
- if (str.length != 2) {
- continue;
- }
- return Long.parseLong(str[0]);
- }
- DataNode.LOG.warn("Block " + blockFile +
- " does not have a metafile!");
- return Block.GRANDFATHER_GENERATION_STAMP;
- }
-
/**
* Populate the given blockSet with any child blocks
* found at this node.
@@ -223,7 +225,7 @@ public class FSDataset implements FSCons
if (blockFiles != null) {
for (int i = 0; i < blockFiles.length; i++) {
if (Block.isBlockFilename(blockFiles[i])) {
- long genStamp = getGenerationStampFromFile(blockFiles,
+ long genStamp = FSDataset.getGenerationStampFromFile(blockFiles,
blockFiles[i]);
blockSet.add(new Block(blockFiles[i], blockFiles[i].length(),
genStamp));
@@ -247,7 +249,7 @@ public class FSDataset implements FSCons
File blockFiles[] = dir.listFiles();
for (int i = 0; i < blockFiles.length; i++) {
if (Block.isBlockFilename(blockFiles[i])) {
- long genStamp = getGenerationStampFromFile(blockFiles, blockFiles[i]);
+ long genStamp = FSDataset.getGenerationStampFromFile(blockFiles, blockFiles[i]);
Block block = new Block(blockFiles[i], blockFiles[i].length(), genStamp);
blockSet.add(new BlockAndFile(blockFiles[i].getAbsoluteFile(), block));
}
@@ -265,7 +267,7 @@ public class FSDataset implements FSCons
if (blockFiles != null) {
for (int i = 0; i < blockFiles.length; i++) {
if (Block.isBlockFilename(blockFiles[i])) {
- long genStamp = getGenerationStampFromFile(blockFiles,
+ long genStamp = FSDataset.getGenerationStampFromFile(blockFiles,
blockFiles[i]);
volumeMap.put(new Block(blockFiles[i], blockFiles[i].length(),
genStamp), new DatanodeBlockInfo(volume, blockFiles[i]));
@@ -383,7 +385,7 @@ public class FSDataset implements FSCons
// should not be deleted.
blocksBeingWritten = new File(parent, "blocksBeingWritten");
if (blocksBeingWritten.exists()) {
- if (supportAppends) {
+ if (supportAppends) {
recoverBlocksBeingWritten(blocksBeingWritten);
} else {
FileUtil.fullyDelete(blocksBeingWritten);
@@ -515,6 +517,35 @@ public class FSDataset implements FSCons
void getBlockInfo(TreeSet<Block> blockSet) {
dataDir.getBlockInfo(blockSet);
}
+
+ void getBlocksBeingWrittenInfo(TreeSet<Block> blockSet) {
+ if (blocksBeingWritten == null) {
+ return;
+ }
+
+ File[] blockFiles = blocksBeingWritten.listFiles();
+ if (blockFiles == null) {
+ return;
+ }
+
+ for (int i = 0; i < blockFiles.length; i++) {
+ if (!blockFiles[i].isDirectory()) {
+ // get each block in the blocksBeingWritten direcotry
+ if (Block.isBlockFilename(blockFiles[i])) {
+ long genStamp =
+ FSDataset.getGenerationStampFromFile(blockFiles, blockFiles[i]);
+ Block block =
+ new Block(blockFiles[i], blockFiles[i].length(), genStamp);
+
+ // add this block to block set
+ blockSet.add(block);
+ if (DataNode.LOG.isDebugEnabled()) {
+ DataNode.LOG.debug("recoverBlocksBeingWritten for block " + block);
+ }
+ }
+ }
+ }
+ }
void getVolumeMap(HashMap<Block, DatanodeBlockInfo> volumeMap) {
dataDir.getVolumeMap(volumeMap, this);
@@ -546,8 +577,6 @@ public class FSDataset implements FSCons
if (DataNode.LOG.isDebugEnabled()) {
DataNode.LOG.debug("recoverBlocksBeingWritten for block " + b.block);
}
- DataNode.getDataNode().notifyNamenodeReceivedBlock(b.block,
- DataNode.EMPTY_DEL_HINT);
}
}
@@ -656,6 +685,18 @@ public class FSDataset implements FSCons
volumes[idx].getVolumeMap(volumeMap);
}
}
+
+ synchronized void getBlocksBeingWrittenInfo(TreeSet<Block> blockSet) {
+ long startTime = System.currentTimeMillis();
+
+ for (int idx = 0; idx < volumes.length; idx++) {
+ volumes[idx].getBlocksBeingWrittenInfo(blockSet);
+ }
+
+ long scanTime = (System.currentTimeMillis() - startTime)/1000;
+ DataNode.LOG.info("Finished generating blocks being written report for " +
+ volumes.length + " volumes in " + scanTime + " seconds");
+ }
/**
* goes over all the volumes and checkDir eachone of them
@@ -1511,6 +1552,20 @@ public class FSDataset implements FSCons
}
/**
+ * Return a table of blocks being written data
+ */
+ public Block[] getBlocksBeingWrittenReport() {
+ TreeSet<Block> blockSet = new TreeSet<Block>();
+ volumes.getBlocksBeingWrittenInfo(blockSet);
+ Block blockTable[] = new Block[blockSet.size()];
+ int i = 0;
+ for (Iterator<Block> it = blockSet.iterator(); it.hasNext(); i++) {
+ blockTable[i] = it.next();
+ }
+ return blockTable;
+ }
+
+ /**
* Return a table of block data
*/
public Block[] getBlockReport() {
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java Fri Sep 9 19:27:21 2011
@@ -233,6 +233,12 @@ public interface FSDatasetInterface exte
* @return - the block report - the full list of blocks stored
*/
public Block[] getBlockReport();
+
+ /**
+ * Returns the blocks being written report
+ * @return - the blocks being written report
+ */
+ public Block[] getBlocksBeingWrittenReport();
/**
* Is the block valid?
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Sep 9 19:27:21 2011
@@ -3303,7 +3303,89 @@ public class FSNamesystem implements FSC
allAlive = !foundDead;
}
}
-
+
+ /**
+ * Log a rejection of an addStoredBlock RPC, invalidate the reported block,
+ * and return it.
+ */
+ private Block rejectAddStoredBlock(Block block, DatanodeDescriptor node,
+ String msg) {
+ NameNode.stateChangeLog.info("BLOCK* NameSystem.addStoredBlock: "
+ + "addStoredBlock request received for " + block + " on "
+ + node.getName() + " size " + block.getNumBytes()
+ + " but was rejected: " + msg);
+ addToInvalidates(block, node);
+ return block;
+ }
+
+ /**
+ * It will update the targets for INodeFileUnderConstruction
+ *
+ * @param nodeID
+ * - DataNode ID
+ * @param blocksBeingWritten
+ * - list of blocks which are still inprogress.
+ * @throws IOException
+ */
+ public synchronized void processBlocksBeingWrittenReport(DatanodeID nodeID,
+ BlockListAsLongs blocksBeingWritten) throws IOException {
+ DatanodeDescriptor dataNode = getDatanode(nodeID);
+ if (dataNode == null) {
+ throw new IOException("ProcessReport from unregistered node: "
+ + nodeID.getName());
+ }
+
+ // Check if this datanode should actually be shutdown instead.
+ if (shouldNodeShutdown(dataNode)) {
+ setDatanodeDead(dataNode);
+ throw new DisallowedDatanodeException(dataNode);
+ }
+
+ Block block = new Block();
+
+ for (int i = 0; i < blocksBeingWritten.getNumberOfBlocks(); i++) {
+ block.set(blocksBeingWritten.getBlockId(i), blocksBeingWritten
+ .getBlockLen(i), blocksBeingWritten.getBlockGenStamp(i));
+
+ BlockInfo storedBlock = blocksMap.getStoredBlockWithoutMatchingGS(block);
+
+ if (storedBlock == null) {
+ rejectAddStoredBlock(new Block(block), dataNode,
+ "Block not in blockMap with any generation stamp");
+ continue;
+ }
+
+ INodeFile inode = storedBlock.getINode();
+ if (inode == null) {
+ rejectAddStoredBlock(new Block(block), dataNode,
+ "Block does not correspond to any file");
+ continue;
+ }
+
+ boolean underConstruction = inode.isUnderConstruction();
+ boolean isLastBlock = inode.getLastBlock() != null
+ && inode.getLastBlock().getBlockId() == block.getBlockId();
+
+ // Must be the last block of a file under construction,
+ if (!underConstruction) {
+ rejectAddStoredBlock(new Block(block), dataNode,
+ "Reported as block being written but is a block of closed file.");
+ continue;
+ }
+
+ if (!isLastBlock) {
+ rejectAddStoredBlock(new Block(block), dataNode,
+ "Reported as block being written but not the last block of "
+ + "an under-construction file.");
+ continue;
+ }
+
+ INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) inode;
+ pendingFile.addTarget(dataNode);
+ incrementSafeBlockCount(pendingFile.getTargets().length);
+ }
+ }
+
/**
* The given node is reporting all its blocks. Use this info to
* update the (machine-->blocklist) and (block-->machinelist) tables.
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Fri Sep 9 19:27:21 2011
@@ -206,4 +206,13 @@ class INodeFile extends INode {
blocks, getPermissionStatus(),
clientName, clientMachine, clientNode);
}
+
+ /**
+ * Return the last block in this file, or null if there are no blocks.
+ */
+ Block getLastBlock() {
+ if (this.blocks == null || this.blocks.length == 0)
+ return null;
+ return this.blocks[this.blocks.length - 1];
+ }
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Sep 9 19:27:21 2011
@@ -932,6 +932,21 @@ public class NameNode implements ClientP
return DatanodeCommand.FINALIZE;
return null;
}
+
+ /**
+ * add new replica blocks to the Inode to target mapping
+ * also add the Inode file to DataNodeDesc
+ */
+ public void blocksBeingWrittenReport(DatanodeRegistration nodeReg,
+ long[] blocks) throws IOException {
+ verifyRequest(nodeReg);
+ BlockListAsLongs blist = new BlockListAsLongs(blocks);
+ namesystem.processBlocksBeingWrittenReport(nodeReg, blist);
+
+ stateChangeLog.info("*BLOCK* NameNode.blocksBeingWrittenReport: "
+ +"from "+nodeReg.getName()+" "+blocks.length +" blocks");
+
+ }
public void blockReceived(DatanodeRegistration nodeReg,
Block blocks[],
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java Fri Sep 9 19:27:21 2011
@@ -110,6 +110,17 @@ public interface DatanodeProtocol extend
*/
public DatanodeCommand blockReport(DatanodeRegistration registration,
long[] blocks) throws IOException;
+
+ /**
+ * blocksBeingWrittenReport() tells the NameNode about the blocks-being-
+ * written information
+ *
+ * @param registration
+ * @param blocks
+ * @throws IOException
+ */
+ public void blocksBeingWrittenReport(DatanodeRegistration registration,
+ long[] blocks) throws IOException;
/**
* blockReceived() allows the DataNode to tell the NameNode about
Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java Fri Sep 9 19:27:21 2011
@@ -572,12 +572,23 @@ public class MiniDFSCluster {
}
/**
+ * Restart namenode. Waits for exit from safemode.
+ */
+ public synchronized void restartNameNode()
+ throws IOException {
+ restartNameNode(true);
+ }
+
+ /**
* Restart namenode.
*/
- public synchronized void restartNameNode() throws IOException {
+ public synchronized void restartNameNode(boolean waitSafemodeExit)
+ throws IOException {
shutdownNameNode();
nameNode = NameNode.createNameNode(new String[] {}, conf);
- waitClusterUp();
+ if (waitSafemodeExit) {
+ waitClusterUp();
+ }
System.out.println("Restarted the namenode");
int failedCount = 0;
while (true) {
Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1167333&r1=1167332&r2=1167333&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Fri Sep 9 19:27:21 2011
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -683,4 +682,9 @@ public class SimulatedFSDataset impleme
public boolean hasEnoughResource() {
return true;
}
+
+ @Override
+ public Block[] getBlocksBeingWrittenReport() {
+ return null;
+ }
}