You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2016/01/05 20:52:14 UTC
[14/50] [abbrv] hadoop git commit: [partial-ns] Implement fsync().
[partial-ns] Implement fsync().
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f09c483
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f09c483
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f09c483
Branch: refs/heads/feature-HDFS-8286
Commit: 7f09c483706c75b9207e008b7891add133e76da4
Parents: 5fe5b9a
Author: Haohui Mai <wh...@apache.org>
Authored: Mon May 25 19:51:53 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Jun 12 13:56:58 2015 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 12 ++++++
.../hdfs/server/namenode/FSDirWriteFileOp.java | 28 +++++++------
.../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +--
.../hdfs/server/namenode/FSNamesystem.java | 41 +++++++++++++-------
.../namenode/FileUnderConstructionFeature.java | 18 ---------
.../hdfs/server/namenode/RWTransaction.java | 10 +++++
6 files changed, 68 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f09c483/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index afd8bb1..e6eb635 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -360,6 +360,18 @@ public class BlockManager {
LOG.info("maxNumBlocksToLog = " + maxNumBlocksToLog);
}
+ public void updateLastBlockLength(Block block, long blockLength) {
+ BlockInfoContiguous lastBlock = getStoredBlock(block);
+ assert (lastBlock != null) : "The last block " + block + " is null when updating its length";
+ assert (lastBlock instanceof BlockInfoContiguousUnderConstruction)
+ : "The last block " + block
+ + " is not a BlockInfoUnderConstruction when updating its length";
+ assert !lastBlock.isComplete();
+ BlockInfoContiguousUnderConstruction uc =
+ (BlockInfoContiguousUnderConstruction) lastBlock;
+ uc.setNumBytes(blockLength);
+ }
+
private static BlockTokenSecretManager createBlockTokenSecretManager(
final Configuration conf) throws IOException {
final boolean isEnabled = conf.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f09c483/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 4cb7f2c..9e9aa93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -100,13 +100,18 @@ class FSDirWriteFileOp {
*/
static void persistBlocks(
FSDirectory fsd, String path, INodeFile file, boolean logRetryCache) {
- assert fsd.getFSNamesystem().hasWriteLock();
- Preconditions.checkArgument(file.isUnderConstruction());
- fsd.getEditLog().logUpdateBlocks(path, file, logRetryCache);
+ throw new IllegalStateException("Unimplemented");
+ }
+
+ static void persistBlocks(
+ RWTransaction tx, String path, FlatINode inode) {
+ FlatINodeFileFeature f = inode.feature(FlatINodeFileFeature.class);
+ Preconditions.checkArgument(f != null && f.inConstruction());
+ tx.logUpdateBlocks(path, f);
if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("persistBlocks: " + path
- + " with " + file.getBlocks().length + " blocks is persisted to" +
- " the file system");
+ NameNode.stateChangeLog.debug(
+ "persistBlocks: " + path + " with " + f.numBlocks() + " " +
+ "blocks is persisted to the file system");
}
}
@@ -528,8 +533,7 @@ class FSDirWriteFileOp {
final INodeFile newNode;
assert fsd.hasWriteLock();
if (underConstruction) {
- newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication,
+ newNode = newINodeFile(id, permissions, modificationTime, modificationTime, replication,
preferredBlockSize,
storagePolicyId);
newNode.toUnderConstruction(clientName, clientMachine);
@@ -553,7 +557,7 @@ class FSDirWriteFileOp {
return newNode;
}
} catch (IOException e) {
- if(NameNode.stateChangeLog.isDebugEnabled()) {
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug(
"DIR* FSDirectory.unprotectedAddFile: exception when add "
+ existing.getPath() + " to the file system", e);
@@ -685,9 +689,9 @@ class FSDirWriteFileOp {
return new FileState(inode, src);
}
- static boolean completeFile(FSNamesystem fsn, FSPermissionChecker pc,
- final String srcArg, String holder, ExtendedBlock last, long fileId)
- throws IOException {
+ static boolean completeFile(
+ FSNamesystem fsn, FSPermissionChecker pc, final String srcArg,
+ String holder, ExtendedBlock last, long fileId) throws IOException {
String src = srcArg;
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f09c483/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 739b2d4..a6df981 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -799,11 +799,11 @@ public class FSEditLog implements LogsPurgeable {
logEdit(op);
}
- public void logUpdateBlocks(String path, INodeFile file, boolean toLogRpcIds) {
- Preconditions.checkArgument(file.isUnderConstruction());
+ public void logUpdateBlocks(String path, Block[] blocks, boolean
+ toLogRpcIds) {
UpdateBlocksOp op = UpdateBlocksOp.getInstance(cache.get())
.setPath(path)
- .setBlocks(file.getBlocks());
+ .setBlocks(blocks);
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f09c483/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index abb316e..9432bc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -143,7 +143,6 @@ import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
@@ -3354,31 +3353,45 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws IOException {
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
waitForLoadingFSImage();
writeLock();
- try {
+ try (RWTransaction tx = dir.newRWTransaction()) {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot fsync file " + src);
- src = dir.resolvePath(pc, src, pathComponents);
- final INode inode;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
+ tx.begin();
+ Resolver.Result paths = Resolver.resolve(tx, src);
+ if (true || fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
// Older clients may not have given us an inode ID to work with.
// In this case, we have to try to resolve the path and hope it
// hasn't changed or been deleted since the file was opened for write.
- inode = dir.getINode(src);
+ paths = Resolver.resolve(tx, src);
} else {
- inode = dir.getInode(fileId);
- if (inode != null) src = inode.getFullPathName();
- }
- final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
+ // Newer clients pass the inode ID, so we can just get the inode
+ // directly.
+ paths = Resolver.resolveById(tx, fileId);
+ }
+ if (paths.invalidPath()) {
+ throw new InvalidPathException(src);
+ } else if (paths.notFound()) {
+ throw new FileNotFoundException(src);
+ }
+ FlatINode inode = paths.inodesInPath().getLastINode();
+ checkLease(src, clientName, inode);
+ FlatINodeFileFeature f = inode.feature(FlatINodeFileFeature.class);
+ FlatINodeFileFeature.Builder newFile = new FlatINodeFileFeature.Builder
+ ().mergeFrom(f);
if (lastBlockLength > 0) {
- pendingFile.getFileUnderConstructionFeature().updateLengthOfLastBlock(
- pendingFile, lastBlockLength);
+ blockManager.updateLastBlockLength(f.lastBlock(), lastBlockLength);
+ Block newLastBlock = f.lastBlock();
+ newLastBlock.setNumBytes(lastBlockLength);
+ newFile.block(f.numBlocks() - 1, newLastBlock);
}
- FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, false);
+
+ ByteString newINode = new FlatINode.Builder().mergeFrom(inode)
+ .replaceFeature(FlatINodeFileFeature.wrap(newFile.build())).build();
+ FSDirWriteFileOp.persistBlocks(tx, src, FlatINode.wrap(newINode));
} finally {
writeUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f09c483/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
index 1ebdde6..603dfc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
@@ -50,24 +50,6 @@ public class FileUnderConstructionFeature implements INode.Feature {
}
/**
- * Update the length for the last block
- *
- * @param lastBlockLength
- * The length of the last block reported from client
- * @throws IOException
- */
- void updateLengthOfLastBlock(INodeFile f, long lastBlockLength)
- throws IOException {
- BlockInfoContiguous lastBlock = f.getLastBlock();
- assert (lastBlock != null) : "The last block for path "
- + f.getFullPathName() + " is null when updating its length";
- assert (lastBlock instanceof BlockInfoContiguousUnderConstruction)
- : "The last block for path " + f.getFullPathName()
- + " is not a BlockInfoUnderConstruction when updating its length";
- lastBlock.setNumBytes(lastBlockLength);
- }
-
- /**
* When deleting a file in the current fs directory, and the file is contained
* in a snapshot, we should delete the last block if it's under construction
* and its size is 0.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f09c483/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RWTransaction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RWTransaction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RWTransaction.java
index b0952a8..9b28625 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RWTransaction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RWTransaction.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.protobuf.ByteString;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.Block;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -172,4 +173,13 @@ class RWTransaction extends Transaction {
public void logSetPermissions(String src, FsPermission permission) {
fsd.getEditLog().logSetPermissions(src, permission);
}
+
+ public void logUpdateBlocks(String path, FlatINodeFileFeature file) {
+ Block[] blocks = new Block[file.numBlocks()];
+ int i = 0;
+ for (Block b : file.blocks()) {
+ blocks[i++] = b;
+ }
+ fsd.getEditLog().logUpdateBlocks(path, blocks, false);
+ }
}