You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/12/13 00:13:56 UTC
[2/2] hadoop git commit: HDFS-7059. Avoid resolving path multiple
times. Contributed by Jing Zhao.
HDFS-7059. Avoid resolving path multiple times. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c78e3a7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c78e3a7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c78e3a7c
Branch: refs/heads/trunk
Commit: c78e3a7cdd10c40454e9acb06986ba6d8573cb19
Parents: 7784b10
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Dec 12 14:15:06 2014 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Dec 12 15:13:35 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 27 +--
.../hdfs/server/namenode/FSDirMkdirOp.java | 33 ++-
.../hdfs/server/namenode/FSDirRenameOp.java | 173 ++++++-------
.../server/namenode/FSDirStatAndListingOp.java | 34 +--
.../hdfs/server/namenode/FSDirXAttrOp.java | 11 +-
.../hdfs/server/namenode/FSDirectory.java | 243 +++++++------------
.../hdfs/server/namenode/FSEditLogLoader.java | 37 +--
.../hdfs/server/namenode/FSImageFormat.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 200 ++++++++-------
.../hdfs/server/namenode/INodesInPath.java | 137 +++++++----
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../hdfs/server/namenode/TestLeaseManager.java | 2 +
.../server/namenode/TestSnapshotPathINodes.java | 64 ++---
.../snapshot/TestOpenFilesWithSnapshot.java | 3 +-
.../snapshot/TestSnapshotReplication.java | 11 +-
18 files changed, 473 insertions(+), 526 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d635400..eeedb0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -453,6 +453,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7463. Simplify FSNamesystem#getBlockLocationsUpdateTimes. (wheat9)
+ HDFS-7509. Avoid resolving path multiple times. (jing9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index c2dee20..0d2b34c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -46,7 +46,7 @@ class FSDirAclOp {
INodesInPath iip = fsd.getINodesInPath4Write(
FSDirectory.normalizePath(src), true);
fsd.checkOwner(pc, iip);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
List<AclEntry> newAcl = AclTransformation.mergeAclEntries(
@@ -72,7 +72,7 @@ class FSDirAclOp {
INodesInPath iip = fsd.getINodesInPath4Write(
FSDirectory.normalizePath(src), true);
fsd.checkOwner(pc, iip);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
List<AclEntry> newAcl = AclTransformation.filterAclEntriesByAclSpec(
@@ -97,7 +97,7 @@ class FSDirAclOp {
INodesInPath iip = fsd.getINodesInPath4Write(
FSDirectory.normalizePath(src), true);
fsd.checkOwner(pc, iip);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
List<AclEntry> newAcl = AclTransformation.filterDefaultAclEntries(
@@ -121,7 +121,7 @@ class FSDirAclOp {
try {
INodesInPath iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
- unprotectedRemoveAcl(fsd, src);
+ unprotectedRemoveAcl(fsd, iip);
} finally {
fsd.writeUnlock();
}
@@ -168,7 +168,7 @@ class FSDirAclOp {
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
}
- INode inode = FSDirectory.resolveLastINode(srcs, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getPathSnapshotId();
List<AclEntry> acl = AclStorage.readINodeAcl(inode, snapshotId);
FsPermission fsPermission = inode.getFsPermission(snapshotId);
@@ -185,16 +185,17 @@ class FSDirAclOp {
static List<AclEntry> unprotectedSetAcl(
FSDirectory fsd, String src, List<AclEntry> aclSpec)
throws IOException {
+ assert fsd.hasWriteLock();
+ final INodesInPath iip = fsd.getINodesInPath4Write(
+ FSDirectory.normalizePath(src), true);
+
// ACL removal is logged to edits as OP_SET_ACL with an empty list.
if (aclSpec.isEmpty()) {
- unprotectedRemoveAcl(fsd, src);
+ unprotectedRemoveAcl(fsd, iip);
return AclFeature.EMPTY_ENTRY_LIST;
}
- assert fsd.hasWriteLock();
- INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath
- (src), true);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
List<AclEntry> newAcl = AclTransformation.replaceAclEntries(existingAcl,
@@ -212,12 +213,10 @@ class FSDirAclOp {
}
}
- private static void unprotectedRemoveAcl(FSDirectory fsd, String src)
+ private static void unprotectedRemoveAcl(FSDirectory fsd, INodesInPath iip)
throws IOException {
assert fsd.hasWriteLock();
- INodesInPath iip = fsd.getINodesInPath4Write(
- FSDirectory.normalizePath(src), true);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
AclFeature f = inode.getAclFeature();
if (f == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index c8c5cb2..7e62d2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -50,8 +50,7 @@ class FSDirMkdirOp {
throw new InvalidPathException(src);
}
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath
- (src);
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
INodesInPath iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
@@ -72,7 +71,7 @@ class FSDirMkdirOp {
// create multiple inodes.
fsn.checkFsObjectLimit();
- if (!mkdirsRecursively(fsd, src, permissions, false, now())) {
+ if (mkdirsRecursively(fsd, iip, permissions, false, now()) == null) {
throw new IOException("Failed to create directory: " + src);
}
}
@@ -97,33 +96,34 @@ class FSDirMkdirOp {
* If ancestor directories do not exist, automatically create them.
* @param fsd FSDirectory
- * @param src string representation of the path to the directory
+ * @param iip the INodesInPath instance containing all the existing INodes
+ * and null elements for non-existing components in the path
* @param permissions the permission of the directory
* @param inheritPermission
* if the permission of the directory should inherit from its parent or not.
* u+wx is implicitly added to the automatically created directories,
* and to the given directory if inheritPermission is true
* @param now creation time
- * @return true if the operation succeeds false otherwise
+ * @return non-null INodesInPath instance if operation succeeds
* @throws QuotaExceededException if directory creation violates
* any quota limit
* @throws UnresolvedLinkException if a symlink is encountered in src.
* @throws SnapshotAccessControlException if path is in RO snapshot
*/
- static boolean mkdirsRecursively(
- FSDirectory fsd, String src, PermissionStatus permissions,
- boolean inheritPermission, long now)
+ static INodesInPath mkdirsRecursively(FSDirectory fsd, INodesInPath iip,
+ PermissionStatus permissions, boolean inheritPermission, long now)
throws FileAlreadyExistsException, QuotaExceededException,
UnresolvedLinkException, SnapshotAccessControlException,
AclException {
- src = FSDirectory.normalizePath(src);
- String[] names = INode.getPathNames(src);
- byte[][] components = INode.getPathComponents(names);
- final int lastInodeIndex = components.length - 1;
+ final int lastInodeIndex = iip.length() - 1;
+ final byte[][] components = iip.getPathComponents();
+ final String[] names = new String[components.length];
+ for (int i = 0; i < components.length; i++) {
+ names[i] = DFSUtil.bytes2String(components[i]);
+ }
fsd.writeLock();
try {
- INodesInPath iip = fsd.getExistingPathINodes(components);
if (iip.isSnapshot()) {
throw new SnapshotAccessControlException(
"Modification on RO snapshot is disallowed");
@@ -136,8 +136,7 @@ class FSDirMkdirOp {
for(; i < length && (curNode = iip.getINode(i)) != null; i++) {
pathbuilder.append(Path.SEPARATOR).append(names[i]);
if (!curNode.isDirectory()) {
- throw new FileAlreadyExistsException(
- "Parent path is not a directory: "
+ throw new FileAlreadyExistsException("Parent path is not a directory: "
+ pathbuilder + " " + curNode.getLocalName());
}
}
@@ -181,7 +180,7 @@ class FSDirMkdirOp {
components[i], (i < lastInodeIndex) ? parentPermissions :
permissions, null, now);
if (iip.getINode(i) == null) {
- return false;
+ return null;
}
// Directory creation also count towards FilesCreated
// to match count of FilesDeleted metric.
@@ -197,7 +196,7 @@ class FSDirMkdirOp {
} finally {
fsd.writeUnlock();
}
- return true;
+ return iip;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index c62c88e..e3020ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.util.Time;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -43,9 +44,9 @@ import java.util.Map;
import static org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
import static org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
-import static org.apache.hadoop.util.Time.now;
class FSDirRenameOp {
+ @Deprecated
static RenameOldResult renameToInt(
FSDirectory fsd, final String srcArg, final String dstArg,
boolean logRetryCache)
@@ -67,7 +68,7 @@ class FSDirRenameOp {
src = fsd.resolvePath(pc, src, srcComponents);
dst = fsd.resolvePath(pc, dst, dstComponents);
@SuppressWarnings("deprecation")
- final boolean status = renameToInternal(fsd, pc, src, dst, logRetryCache);
+ final boolean status = renameTo(fsd, pc, src, dst, logRetryCache);
if (status) {
resultingStat = fsd.getAuditFileInfo(dst, false);
}
@@ -116,6 +117,22 @@ class FSDirRenameOp {
}
/**
+ * <br>
+ * Note: This is to be used by {@link FSEditLogLoader} only.
+ * <br>
+ */
+ @Deprecated
+ static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+ long timestamp) throws IOException {
+ if (fsd.isDir(dst)) {
+ dst += Path.SEPARATOR + new Path(src).getName();
+ }
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ return unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp);
+ }
+
+ /**
* Change a path name
*
* @param fsd FSDirectory
@@ -126,24 +143,19 @@ class FSDirRenameOp {
* boolean, Options.Rename...)}
*/
@Deprecated
- static boolean unprotectedRenameTo(
- FSDirectory fsd, String src, String dst, long timestamp)
+ static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+ final INodesInPath srcIIP, final INodesInPath dstIIP, long timestamp)
throws IOException {
assert fsd.hasWriteLock();
- INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
final INode srcInode = srcIIP.getLastINode();
try {
- validateRenameSource(src, srcIIP);
+ validateRenameSource(srcIIP);
} catch (SnapshotException e) {
throw e;
} catch (IOException ignored) {
return false;
}
- if (fsd.isDir(dst)) {
- dst += Path.SEPARATOR + new Path(src).getName();
- }
-
// validate the destination
if (dst.equals(src)) {
return true;
@@ -155,7 +167,6 @@ class FSDirRenameOp {
return false;
}
- INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
if (dstIIP.getLastINode() != null) {
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
"failed to rename " + src + " to " + dst + " because destination " +
@@ -234,8 +245,7 @@ class FSDirRenameOp {
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
src = fsd.resolvePath(pc, src, srcComponents);
dst = fsd.resolvePath(pc, dst, dstComponents);
- renameToInternal(fsd, pc, src, dst, logRetryCache, collectedBlocks,
- options);
+ renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dst, false);
return new AbstractMap.SimpleImmutableEntry<BlocksMapUpdateInfo,
@@ -246,29 +256,44 @@ class FSDirRenameOp {
* @see #unprotectedRenameTo(FSDirectory, String, String, long,
* org.apache.hadoop.fs.Options.Rename...)
*/
- static void renameTo(
- FSDirectory fsd, String src, String dst, long mtime,
- BlocksMapUpdateInfo collectedBlocks, Options.Rename... options)
- throws IOException {
+ static void renameTo(FSDirectory fsd, FSPermissionChecker pc, String src,
+ String dst, BlocksMapUpdateInfo collectedBlocks, boolean logRetryCache,
+ Options.Rename... options) throws IOException {
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ if (fsd.isPermissionEnabled()) {
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ // Check write access to parent of src
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
+ false);
+ // Check write access to ancestor of dst
+ fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
+ false);
+ }
+
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
+ dst);
}
+ final long mtime = Time.now();
fsd.writeLock();
try {
- if (unprotectedRenameTo(fsd, src, dst, mtime, collectedBlocks, options)) {
+ if (unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, mtime,
+ collectedBlocks, options)) {
fsd.getFSNamesystem().incrDeletedFileCount(1);
}
} finally {
fsd.writeUnlock();
}
+ fsd.getEditLog().logRename(src, dst, mtime, logRetryCache, options);
}
/**
* Rename src to dst.
* <br>
* Note: This is to be used by {@link org.apache.hadoop.hdfs.server
- * .namenode.FSEditLog} only.
+ * .namenode.FSEditLogLoader} only.
* <br>
*
* @param fsd FSDirectory
@@ -282,7 +307,9 @@ class FSDirRenameOp {
Options.Rename... options)
throws IOException {
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- boolean ret = unprotectedRenameTo(fsd, src, dst, timestamp,
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ boolean ret = unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp,
collectedBlocks, options);
if (!collectedBlocks.getToDeleteList().isEmpty()) {
fsd.getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
@@ -302,8 +329,8 @@ class FSDirRenameOp {
* @param collectedBlocks blocks to be removed
* @param options Rename options
*/
- static boolean unprotectedRenameTo(
- FSDirectory fsd, String src, String dst, long timestamp,
+ static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+ final INodesInPath srcIIP, final INodesInPath dstIIP, long timestamp,
BlocksMapUpdateInfo collectedBlocks, Options.Rename... options)
throws IOException {
assert fsd.hasWriteLock();
@@ -311,9 +338,8 @@ class FSDirRenameOp {
&& Arrays.asList(options).contains(Options.Rename.OVERWRITE);
final String error;
- final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
final INode srcInode = srcIIP.getLastINode();
- validateRenameSource(src, srcIIP);
+ validateRenameSource(srcIIP);
// validate the destination
if (dst.equals(src)) {
@@ -322,7 +348,6 @@ class FSDirRenameOp {
}
validateDestination(src, dst, srcInode);
- INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
if (dstIIP.length() == 1) {
error = "rename destination cannot be the root";
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
@@ -373,8 +398,8 @@ class FSDirRenameOp {
long removedNum = 0;
try {
if (dstInode != null) { // dst exists remove it
- if ((removedNum = fsd.removeLastINode(dstIIP)) != -1) {
- removedDst = dstIIP.getLastINode();
+ if ((removedNum = fsd.removeLastINode(tx.dstIIP)) != -1) {
+ removedDst = tx.dstIIP.getLastINode();
undoRemoveDst = true;
}
}
@@ -395,13 +420,13 @@ class FSDirRenameOp {
undoRemoveDst = false;
if (removedNum > 0) {
List<INode> removedINodes = new ChunkedArrayList<INode>();
- if (!removedDst.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) {
+ if (!removedDst.isInLatestSnapshot(tx.dstIIP.getLatestSnapshotId())) {
removedDst.destroyAndCollectBlocks(collectedBlocks,
removedINodes);
filesDeleted = true;
} else {
filesDeleted = removedDst.cleanSubtree(
- Snapshot.CURRENT_STATE_ID, dstIIP.getLatestSnapshotId(),
+ Snapshot.CURRENT_STATE_ID, tx.dstIIP.getLatestSnapshotId(),
collectedBlocks, removedINodes, true)
.get(Quota.NAMESPACE) >= 0;
}
@@ -431,7 +456,7 @@ class FSDirRenameOp {
dstParent.asDirectory().undoRename4DstParent(removedDst,
dstIIP.getLatestSnapshotId());
} else {
- fsd.addLastINodeNoQuotaCheck(dstIIP, removedDst);
+ fsd.addLastINodeNoQuotaCheck(tx.dstIIP, removedDst);
}
if (removedDst.isReference()) {
final INodeReference removedDstRef = removedDst.asReference();
@@ -447,59 +472,41 @@ class FSDirRenameOp {
}
/**
- * @see #unprotectedRenameTo(FSDirectory, String, String, long)
* @deprecated Use {@link #renameToInt(FSDirectory, String, String,
* boolean, Options.Rename...)}
*/
@Deprecated
@SuppressWarnings("deprecation")
- private static boolean renameTo(
- FSDirectory fsd, String src, String dst, long mtime)
- throws IOException {
+ private static boolean renameTo(FSDirectory fsd, FSPermissionChecker pc,
+ String src, String dst, boolean logRetryCache) throws IOException {
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ // Check write access to parent of src
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ // Note: We should not be doing this. This is move() not renameTo().
+ final String actualDst = fsd.isDir(dst) ?
+ dst + Path.SEPARATOR + new Path(src).getName() : dst;
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(actualDst, false);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
+ false);
+ // Check write access to ancestor of dst
+ fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
+ null, false);
+ }
+
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
+ dst);
}
+ final long mtime = Time.now();
boolean stat = false;
fsd.writeLock();
try {
- stat = unprotectedRenameTo(fsd, src, dst, mtime);
+ stat = unprotectedRenameTo(fsd, src, actualDst, srcIIP, dstIIP, mtime);
} finally {
fsd.writeUnlock();
}
- return stat;
- }
-
- /**
- * @deprecated See {@link #renameTo(FSDirectory, String, String, long)}
- */
- @Deprecated
- private static boolean renameToInternal(
- FSDirectory fsd, FSPermissionChecker pc, String src, String dst,
- boolean logRetryCache)
- throws IOException {
- if (fsd.isPermissionEnabled()) {
- //We should not be doing this. This is move() not renameTo().
- //but for now,
- //NOTE: yes, this is bad! it's assuming much lower level behavior
- // of rewriting the dst
- String actualdst = fsd.isDir(dst) ? dst + Path.SEPARATOR + new Path
- (src).getName() : dst;
- // Rename does not operates on link targets
- // Do not resolveLink when checking permissions of src and dst
- // Check write access to parent of src
- INodesInPath srcIIP = fsd.getINodesInPath(src, false);
- fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
- false);
- INodesInPath dstIIP = fsd.getINodesInPath(actualdst, false);
- // Check write access to ancestor of dst
- fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
- null, false);
- }
-
- long mtime = now();
- @SuppressWarnings("deprecation")
- final boolean stat = renameTo(fsd, src, dst, mtime);
if (stat) {
fsd.getEditLog().logRename(src, dst, mtime, logRetryCache);
return true;
@@ -507,29 +514,6 @@ class FSDirRenameOp {
return false;
}
- private static void renameToInternal(
- FSDirectory fsd, FSPermissionChecker pc, String src, String dst,
- boolean logRetryCache, BlocksMapUpdateInfo collectedBlocks,
- Options.Rename... options)
- throws IOException {
- if (fsd.isPermissionEnabled()) {
- // Rename does not operates on link targets
- // Do not resolveLink when checking permissions of src and dst
- // Check write access to parent of src
- INodesInPath srcIIP = fsd.getINodesInPath(src, false);
- fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
- false);
- // Check write access to ancestor of dst
- INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
- fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
- false);
- }
-
- long mtime = now();
- renameTo(fsd, src, dst, mtime, collectedBlocks, options);
- fsd.getEditLog().logRename(src, dst, mtime, logRetryCache, options);
- }
-
private static void validateDestination(
String src, String dst, INode srcInode)
throws IOException {
@@ -579,13 +563,13 @@ class FSDirRenameOp {
}
}
- private static void validateRenameSource(String src, INodesInPath srcIIP)
+ private static void validateRenameSource(INodesInPath srcIIP)
throws IOException {
String error;
final INode srcInode = srcIIP.getLastINode();
// validate source
if (srcInode == null) {
- error = "rename source " + src + " is not found.";
+ error = "rename source " + srcIIP.getPath() + " is not found.";
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ error);
throw new FileNotFoundException(error);
@@ -625,8 +609,7 @@ class FSDirRenameOp {
this.dst = dst;
srcChild = srcIIP.getLastINode();
srcChildName = srcChild.getLocalNameBytes();
- isSrcInSnapshot = srcChild.isInLatestSnapshot(srcIIP
- .getLatestSnapshotId());
+ isSrcInSnapshot = srcChild.isInLatestSnapshot(srcIIP.getLatestSnapshotId());
srcChildIsReference = srcChild.isReference();
srcParent = srcIIP.getINode(-2).asDirectory();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 0f94171..5bc790e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -72,14 +72,14 @@ class FSDirStatAndListingOp {
boolean isSuperUser = true;
if (fsd.isPermissionEnabled()) {
- if (fsd.isDir(src)) {
+ if (iip.getLastINode() != null && iip.getLastINode().isDirectory()) {
fsd.checkPathAccess(pc, iip, FsAction.READ_EXECUTE);
} else {
fsd.checkTraverse(pc, iip);
}
isSuperUser = pc.isSuperUser();
}
- return getListing(fsd, src, startAfter, needLocation, isSuperUser);
+ return getListing(fsd, iip, src, startAfter, needLocation, isSuperUser);
}
/**
@@ -131,12 +131,12 @@ class FSDirStatAndListingOp {
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = fsd.getPermissionChecker();
src = fsd.resolvePath(pc, src, pathComponents);
- final INodesInPath iip = fsd.getINodesInPath(src, true);
+ final INodesInPath iip = fsd.getINodesInPath(src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null,
FsAction.READ_EXECUTE);
}
- return getContentSummaryInt(fsd, src);
+ return getContentSummaryInt(fsd, iip);
}
/**
@@ -148,14 +148,15 @@ class FSDirStatAndListingOp {
* that at least this.lsLimit block locations are in the response
*
* @param fsd FSDirectory
+ * @param iip the INodesInPath instance containing all the INodes along the
+ * path
* @param src the directory name
* @param startAfter the name to start listing after
* @param needLocation if block locations are returned
* @return a partial listing starting after startAfter
*/
- private static DirectoryListing getListing(
- FSDirectory fsd, String src, byte[] startAfter, boolean needLocation,
- boolean isSuperUser)
+ private static DirectoryListing getListing(FSDirectory fsd, INodesInPath iip,
+ String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
final boolean isRawPath = FSDirectory.isReservedRawName(src);
@@ -165,9 +166,8 @@ class FSDirStatAndListingOp {
if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
return getSnapshotsListing(fsd, srcs, startAfter);
}
- final INodesInPath inodesInPath = fsd.getINodesInPath(srcs, true);
- final int snapshot = inodesInPath.getPathSnapshotId();
- final INode targetNode = inodesInPath.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final INode targetNode = iip.getLastINode();
if (targetNode == null)
return null;
byte parentStoragePolicy = isSuperUser ?
@@ -178,7 +178,7 @@ class FSDirStatAndListingOp {
return new DirectoryListing(
new HdfsFileStatus[]{createFileStatus(fsd,
HdfsFileStatus.EMPTY_NAME, targetNode, needLocation,
- parentStoragePolicy, snapshot, isRawPath, inodesInPath)}, 0);
+ parentStoragePolicy, snapshot, isRawPath, iip)}, 0);
}
final INodeDirectory dirInode = targetNode.asDirectory();
@@ -196,7 +196,8 @@ class FSDirStatAndListingOp {
cur.getLocalStoragePolicyID():
BlockStoragePolicySuite.ID_UNSPECIFIED;
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), cur,
- needLocation, fsd.getStoragePolicyID(curPolicy, parentStoragePolicy), snapshot, isRawPath, inodesInPath);
+ needLocation, fsd.getStoragePolicyID(curPolicy,
+ parentStoragePolicy), snapshot, isRawPath, iip);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -453,14 +454,13 @@ class FSDirStatAndListingOp {
return perm;
}
- private static ContentSummary getContentSummaryInt(
- FSDirectory fsd, String src) throws IOException {
- String srcs = FSDirectory.normalizePath(src);
+ private static ContentSummary getContentSummaryInt(FSDirectory fsd,
+ INodesInPath iip) throws IOException {
fsd.readLock();
try {
- INode targetNode = fsd.getNode(srcs, false);
+ INode targetNode = iip.getLastINode();
if (targetNode == null) {
- throw new FileNotFoundException("File does not exist: " + srcs);
+ throw new FileNotFoundException("File does not exist: " + iip.getPath());
}
else {
// Make it relinquish locks everytime contentCountLimit entries are
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 303b9e3..47a995d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -191,7 +191,7 @@ class FSDirXAttrOp {
assert fsd.hasWriteLock();
INodesInPath iip = fsd.getINodesInPath4Write(
FSDirectory.normalizePath(src), true);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
@@ -260,8 +260,9 @@ class FSDirXAttrOp {
final EnumSet<XAttrSetFlag> flag)
throws IOException {
assert fsd.hasWriteLock();
- INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src),
+ true);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> newXAttrs = setINodeXAttrs(fsd, existingXAttrs, xAttrs, flag);
@@ -444,8 +445,8 @@ class FSDirXAttrOp {
String srcs = FSDirectory.normalizePath(src);
fsd.readLock();
try {
- INodesInPath iip = fsd.getLastINodeInPath(srcs, true);
- INode inode = FSDirectory.resolveLastINode(src, iip);
+ INodesInPath iip = fsd.getINodesInPath(srcs, true);
+ INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getPathSnapshotId();
return XAttrStorage.readINodeXAttrs(inode, snapshotId);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 81b0eb6..ee9bdd0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -335,8 +335,8 @@ public class FSDirectory implements Closeable {
private static INodeFile newINodeFile(long id, PermissionStatus permissions,
long mtime, long atime, short replication, long preferredBlockSize) {
- return newINodeFile(id, permissions, mtime, atime, replication, preferredBlockSize,
- (byte)0);
+ return newINodeFile(id, permissions, mtime, atime, replication,
+ preferredBlockSize, (byte)0);
}
private static INodeFile newINodeFile(long id, PermissionStatus permissions,
@@ -354,20 +354,21 @@ public class FSDirectory implements Closeable {
* @throws UnresolvedLinkException
* @throws SnapshotAccessControlException
*/
- INodeFile addFile(String path, PermissionStatus permissions,
+ INodeFile addFile(INodesInPath iip, String path, PermissionStatus permissions,
short replication, long preferredBlockSize,
String clientName, String clientMachine)
throws FileAlreadyExistsException, QuotaExceededException,
UnresolvedLinkException, SnapshotAccessControlException, AclException {
long modTime = now();
- INodeFile newNode = newINodeFile(allocateNewInodeId(), permissions, modTime, modTime, replication, preferredBlockSize);
+ INodeFile newNode = newINodeFile(allocateNewInodeId(), permissions, modTime,
+ modTime, replication, preferredBlockSize);
newNode.toUnderConstruction(clientName, clientMachine);
boolean added = false;
writeLock();
try {
- added = addINode(path, newNode);
+ added = addINode(iip, newNode);
} finally {
writeUnlock();
}
@@ -382,8 +383,8 @@ public class FSDirectory implements Closeable {
return newNode;
}
- INodeFile unprotectedAddFile( long id,
- String path,
+ INodeFile unprotectedAddFile(long id,
+ INodesInPath iip,
PermissionStatus permissions,
List<AclEntry> aclEntries,
List<XAttr> xAttrs,
@@ -401,14 +402,13 @@ public class FSDirectory implements Closeable {
newNode = newINodeFile(id, permissions, modificationTime,
modificationTime, replication, preferredBlockSize, storagePolicyId);
newNode.toUnderConstruction(clientName, clientMachine);
-
} else {
newNode = newINodeFile(id, permissions, modificationTime, atime,
replication, preferredBlockSize, storagePolicyId);
}
try {
- if (addINode(path, newNode)) {
+ if (addINode(iip, newNode)) {
if (aclEntries != null) {
AclStorage.updateINodeAcl(newNode, aclEntries,
Snapshot.CURRENT_STATE_ID);
@@ -422,8 +422,8 @@ public class FSDirectory implements Closeable {
} catch (IOException e) {
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug(
- "DIR* FSDirectory.unprotectedAddFile: exception when add " + path
- + " to the file system", e);
+ "DIR* FSDirectory.unprotectedAddFile: exception when add "
+ + iip.getPath() + " to the file system", e);
}
}
return null;
@@ -468,18 +468,18 @@ public class FSDirectory implements Closeable {
* Remove a block from the file.
* @return Whether the block exists in the corresponding file
*/
- boolean removeBlock(String path, INodeFile fileNode, Block block)
- throws IOException {
+ boolean removeBlock(String path, INodesInPath iip, INodeFile fileNode,
+ Block block) throws IOException {
Preconditions.checkArgument(fileNode.isUnderConstruction());
writeLock();
try {
- return unprotectedRemoveBlock(path, fileNode, block);
+ return unprotectedRemoveBlock(path, iip, fileNode, block);
} finally {
writeUnlock();
}
}
- boolean unprotectedRemoveBlock(String path,
+ boolean unprotectedRemoveBlock(String path, INodesInPath iip,
INodeFile fileNode, Block block) throws IOException {
// modify file-> block and blocksMap
// fileNode should be under construction
@@ -496,7 +496,6 @@ public class FSDirectory implements Closeable {
}
// update space consumed
- final INodesInPath iip = getINodesInPath4Write(path, true);
updateCount(iip, 0, -fileNode.getBlockDiskspace(), true);
return true;
}
@@ -638,20 +637,6 @@ public class FSDirectory implements Closeable {
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
}
- /**
- * @param path the file path
- * @return the block size of the file.
- */
- long getPreferredBlockSize(String path) throws IOException {
- readLock();
- try {
- return INodeFile.valueOf(getNode(path, false), path
- ).getPreferredBlockSize();
- } finally {
- readUnlock();
- }
- }
-
void setPermission(String src, FsPermission permission)
throws FileNotFoundException, UnresolvedLinkException,
QuotaExceededException, SnapshotAccessControlException {
@@ -706,28 +691,26 @@ public class FSDirectory implements Closeable {
/**
* Delete the target directory and collect the blocks under it
- *
- * @param src Path of a directory to delete
+ *
+ * @param iip the INodesInPath instance containing all the INodes for the path
* @param collectedBlocks Blocks under the deleted directory
* @param removedINodes INodes that should be removed from {@link #inodeMap}
* @return the number of files that have been removed
*/
- long delete(String src, BlocksMapUpdateInfo collectedBlocks,
+ long delete(INodesInPath iip, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, long mtime) throws IOException {
if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
+ NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + iip.getPath());
}
final long filesRemoved;
writeLock();
try {
- final INodesInPath inodesInPath = getINodesInPath4Write(
- normalizePath(src), false);
- if (!deleteAllowed(inodesInPath, src) ) {
+ if (!deleteAllowed(iip, iip.getPath()) ) {
filesRemoved = -1;
} else {
List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
- FSDirSnapshotOp.checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
- filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
+ FSDirSnapshotOp.checkSnapshot(iip.getLastINode(), snapshottableDirs);
+ filesRemoved = unprotectedDelete(iip, collectedBlocks,
removedINodes, mtime);
namesystem.removeSnapshottableDirs(snapshottableDirs);
}
@@ -863,88 +846,15 @@ public class FSDirectory implements Closeable {
parentPolicy;
}
- INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {
- Preconditions.checkArgument(
- src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
- "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
-
- final String dirPath = normalizePath(src.substring(0,
- src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
-
- final INode node = this.getINode(dirPath);
- if (node != null && node.isDirectory()
- && node.asDirectory().isSnapshottable()) {
- return node;
- }
- return null;
- }
-
- INodesInPath getExistingPathINodes(byte[][] components)
- throws UnresolvedLinkException {
- return INodesInPath.resolve(rootDir, components);
- }
-
- /**
- * Get {@link INode} associated with the file / directory.
- */
- public INode getINode(String src) throws UnresolvedLinkException {
- return getLastINodeInPath(src).getINode(0);
- }
-
- /**
- * Get {@link INode} associated with the file / directory.
- */
- public INodesInPath getLastINodeInPath(String src)
- throws UnresolvedLinkException {
- readLock();
- try {
- return getLastINodeInPath(src, true);
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Get {@link INode} associated with the file / directory.
- */
- public INodesInPath getINodesInPath4Write(String src
- ) throws UnresolvedLinkException, SnapshotAccessControlException {
- readLock();
- try {
- return getINodesInPath4Write(src, true);
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Get {@link INode} associated with the file / directory.
- * @throws SnapshotAccessControlException if path is in RO snapshot
- */
- public INode getINode4Write(String src) throws UnresolvedLinkException,
- SnapshotAccessControlException {
- readLock();
- try {
- return getINode4Write(src, true);
- } finally {
- readUnlock();
- }
- }
-
/**
* Check whether the filepath could be created
* @throws SnapshotAccessControlException if path is in RO snapshot
*/
- boolean isValidToCreate(String src) throws UnresolvedLinkException,
- SnapshotAccessControlException {
+ boolean isValidToCreate(String src, INodesInPath iip)
+ throws SnapshotAccessControlException {
String srcs = normalizePath(src);
- readLock();
- try {
- return srcs.startsWith("/") && !srcs.endsWith("/")
- && getINode4Write(srcs, false) == null;
- } finally {
- readUnlock();
- }
+ return srcs.startsWith("/") && !srcs.endsWith("/") &&
+ iip.getLastINode() == null;
}
/**
@@ -954,7 +864,7 @@ public class FSDirectory implements Closeable {
src = normalizePath(src);
readLock();
try {
- INode node = getNode(src, false);
+ INode node = getINode(src, false);
return node != null && node.isDirectory();
} finally {
readUnlock();
@@ -963,21 +873,21 @@ public class FSDirectory implements Closeable {
/** Updates namespace and diskspace consumed for all
* directories until the parent directory of file represented by path.
- *
- * @param path path for the file.
+ *
+ * @param iip the INodesInPath instance containing all the INodes for
+ * updating quota usage
* @param nsDelta the delta change of namespace
* @param dsDelta the delta change of diskspace
* @throws QuotaExceededException if the new count violates any quota limit
* @throws FileNotFoundException if path does not exist.
*/
- void updateSpaceConsumed(String path, long nsDelta, long dsDelta)
+ void updateSpaceConsumed(INodesInPath iip, long nsDelta, long dsDelta)
throws QuotaExceededException, FileNotFoundException,
UnresolvedLinkException, SnapshotAccessControlException {
writeLock();
try {
- final INodesInPath iip = getINodesInPath4Write(path, false);
if (iip.getLastINode() == null) {
- throw new FileNotFoundException("Path not found: " + path);
+ throw new FileNotFoundException("Path not found: " + iip.getPath());
}
updateCount(iip, nsDelta, dsDelta, true);
} finally {
@@ -1097,17 +1007,15 @@ public class FSDirectory implements Closeable {
/**
* Add the given child to the namespace.
- * @param src The full path name of the child node.
+ * @param iip the INodesInPath instance containing all the ancestral INodes
* @throws QuotaExceededException is thrown if it violates quota limit
*/
- private boolean addINode(String src, INode child)
+ private boolean addINode(INodesInPath iip, INode child)
throws QuotaExceededException, UnresolvedLinkException {
- byte[][] components = INode.getPathComponents(src);
- child.setLocalName(components[components.length-1]);
+ child.setLocalName(iip.getLastLocalName());
cacheName(child);
writeLock();
try {
- final INodesInPath iip = getExistingPathINodes(components);
return addLastINode(iip, child, true);
} finally {
writeUnlock();
@@ -1504,7 +1412,7 @@ public class FSDirectory implements Closeable {
boolean unprotectedSetTimes(String src, long mtime, long atime, boolean force)
throws UnresolvedLinkException, QuotaExceededException {
assert hasWriteLock();
- final INodesInPath i = getLastINodeInPath(src);
+ final INodesInPath i = getINodesInPath(src, true);
return unprotectedSetTimes(i.getLastINode(), mtime, atime, force,
i.getLatestSnapshotId());
}
@@ -1551,24 +1459,24 @@ public class FSDirectory implements Closeable {
/**
* Add the specified path into the namespace.
*/
- INodeSymlink addSymlink(long id, String path, String target,
+ INodeSymlink addSymlink(INodesInPath iip, long id, String target,
long mtime, long atime, PermissionStatus perm)
throws UnresolvedLinkException, QuotaExceededException {
writeLock();
try {
- return unprotectedAddSymlink(id, path, target, mtime, atime, perm);
+ return unprotectedAddSymlink(iip, id, target, mtime, atime, perm);
} finally {
writeUnlock();
}
}
- INodeSymlink unprotectedAddSymlink(long id, String path, String target,
+ INodeSymlink unprotectedAddSymlink(INodesInPath iip, long id, String target,
long mtime, long atime, PermissionStatus perm)
throws UnresolvedLinkException, QuotaExceededException {
assert hasWriteLock();
final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
target);
- return addINode(path, symlink) ? symlink : null;
+ return addINode(iip, symlink) ? symlink : null;
}
boolean isInAnEZ(INodesInPath iip)
@@ -1704,11 +1612,10 @@ public class FSDirectory implements Closeable {
}
}
- static INode resolveLastINode(String src, INodesInPath iip)
- throws FileNotFoundException {
+ static INode resolveLastINode(INodesInPath iip) throws FileNotFoundException {
INode inode = iip.getLastINode();
if (inode == null) {
- throw new FileNotFoundException("cannot find " + src);
+ throw new FileNotFoundException("cannot find " + iip.getPath());
}
return inode;
}
@@ -1885,36 +1792,62 @@ public class FSDirectory implements Closeable {
return path.toString();
}
- /** @return the {@link INodesInPath} containing only the last inode. */
- INodesInPath getLastINodeInPath(
- String path, boolean resolveLink) throws UnresolvedLinkException {
- return INodesInPath.resolve(rootDir, INode.getPathComponents(path), 1,
- resolveLink);
+ INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {
+ Preconditions.checkArgument(
+ src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
+ "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
+
+ final String dirPath = normalizePath(src.substring(0,
+ src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
+
+ final INode node = this.getINode(dirPath);
+ if (node != null && node.isDirectory()
+ && node.asDirectory().isSnapshottable()) {
+ return node;
+ }
+ return null;
+ }
+
+ INodesInPath getExistingPathINodes(byte[][] components)
+ throws UnresolvedLinkException {
+ return INodesInPath.resolve(rootDir, components, false);
+ }
+
+ /**
+ * Get {@link INode} associated with the file / directory.
+ */
+ public INodesInPath getINodesInPath4Write(String src)
+ throws UnresolvedLinkException, SnapshotAccessControlException {
+ return getINodesInPath4Write(src, true);
+ }
+
+ /**
+ * Get {@link INode} associated with the file / directory.
+ * @throws SnapshotAccessControlException if path is in RO snapshot
+ */
+ public INode getINode4Write(String src) throws UnresolvedLinkException,
+ SnapshotAccessControlException {
+ return getINodesInPath4Write(src, true).getLastINode();
}
/** @return the {@link INodesInPath} containing all inodes in the path. */
- INodesInPath getINodesInPath(String path, boolean resolveLink
- ) throws UnresolvedLinkException {
+ public INodesInPath getINodesInPath(String path, boolean resolveLink)
+ throws UnresolvedLinkException {
final byte[][] components = INode.getPathComponents(path);
- return INodesInPath.resolve(rootDir, components, components.length,
- resolveLink);
+ return INodesInPath.resolve(rootDir, components, resolveLink);
}
/** @return the last inode in the path. */
- INode getNode(String path, boolean resolveLink)
- throws UnresolvedLinkException {
- return getLastINodeInPath(path, resolveLink).getINode(0);
+ INode getINode(String path, boolean resolveLink)
+ throws UnresolvedLinkException {
+ return getINodesInPath(path, resolveLink).getLastINode();
}
/**
- * @return the INode of the last component in src, or null if the last
- * component does not exist.
- * @throws UnresolvedLinkException if symlink can't be resolved
- * @throws SnapshotAccessControlException if path is in RO snapshot
+ * Get {@link INode} associated with the file / directory.
*/
- INode getINode4Write(String src, boolean resolveLink)
- throws UnresolvedLinkException, SnapshotAccessControlException {
- return getINodesInPath4Write(src, resolveLink).getLastINode();
+ public INode getINode(String src) throws UnresolvedLinkException {
+ return getINode(src, true);
}
/**
@@ -1926,7 +1859,7 @@ public class FSDirectory implements Closeable {
throws UnresolvedLinkException, SnapshotAccessControlException {
final byte[][] components = INode.getPathComponents(src);
INodesInPath inodesInPath = INodesInPath.resolve(rootDir, components,
- components.length, resolveLink);
+ resolveLink);
if (inodesInPath.isSnapshot()) {
throw new SnapshotAccessControlException(
"Modification on a read-only snapshot is disallowed");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 2721f85..833b9db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -342,7 +342,7 @@ public class FSEditLogLoader {
// 3. OP_ADD to open file for append
// See if the file already exists (persistBlocks call)
- final INodesInPath iip = fsDir.getINodesInPath(path, true);
+ INodesInPath iip = fsDir.getINodesInPath(path, true);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path, true);
if (oldFile != null && addCloseOp.overwrite) {
// This is OP_ADD with overwrite
@@ -361,11 +361,12 @@ public class FSEditLogLoader {
inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion,
lastInodeId);
newFile = fsDir.unprotectedAddFile(inodeId,
- path, addCloseOp.permissions, addCloseOp.aclEntries,
+ iip, addCloseOp.permissions, addCloseOp.aclEntries,
addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
addCloseOp.blockSize, true, addCloseOp.clientName,
addCloseOp.clientMachine, addCloseOp.storagePolicyId);
+ iip = INodesInPath.replace(iip, iip.length() - 1, newFile);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
// add the op into retry cache if necessary
@@ -384,10 +385,10 @@ public class FSEditLogLoader {
FSNamesystem.LOG.debug("Reopening an already-closed file " +
"for append");
}
- LocatedBlock lb = fsNamesys.prepareFileForWrite(path,
- oldFile, addCloseOp.clientName, addCloseOp.clientMachine, false, iip.getLatestSnapshotId(), false);
- newFile = INodeFile.valueOf(fsDir.getINode(path),
- path, true);
+ // Note we do not replace the INodeFile when converting it to
+ // under-construction
+ LocatedBlock lb = fsNamesys.prepareFileForWrite(path, iip,
+ addCloseOp.clientName, addCloseOp.clientMachine, false, false);
// add the op into retry cache is necessary
if (toAddRetryCache) {
@@ -408,7 +409,7 @@ public class FSEditLogLoader {
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- updateBlocks(fsDir, addCloseOp, newFile);
+ updateBlocks(fsDir, addCloseOp, iip, newFile);
break;
}
case OP_CLOSE: {
@@ -422,13 +423,13 @@ public class FSEditLogLoader {
" clientMachine " + addCloseOp.clientMachine);
}
- final INodesInPath iip = fsDir.getLastINodeInPath(path);
- final INodeFile file = INodeFile.valueOf(iip.getINode(0), path);
+ final INodesInPath iip = fsDir.getINodesInPath(path, true);
+ final INodeFile file = INodeFile.valueOf(iip.getLastINode(), path);
// Update the salient file attributes.
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- updateBlocks(fsDir, addCloseOp, file);
+ updateBlocks(fsDir, addCloseOp, iip, file);
// Now close the file
if (!file.isUnderConstruction() &&
@@ -455,10 +456,10 @@ public class FSEditLogLoader {
FSNamesystem.LOG.debug(op.opCode + ": " + path +
" numblocks : " + updateOp.blocks.length);
}
- INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(path),
- path);
+ INodesInPath iip = fsDir.getINodesInPath(path, true);
+ INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
- updateBlocks(fsDir, updateOp, oldFile);
+ updateBlocks(fsDir, updateOp, iip, oldFile);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
@@ -587,8 +588,10 @@ public class FSEditLogLoader {
SymlinkOp symlinkOp = (SymlinkOp)op;
inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion,
lastInodeId);
- fsDir.unprotectedAddSymlink(inodeId,
- renameReservedPathsOnUpgrade(symlinkOp.path, logVersion),
+ final String path = renameReservedPathsOnUpgrade(symlinkOp.path,
+ logVersion);
+ final INodesInPath iip = fsDir.getINodesInPath(path, false);
+ fsDir.unprotectedAddSymlink(iip, inodeId,
symlinkOp.value, symlinkOp.mtime, symlinkOp.atime,
symlinkOp.permissionStatus);
@@ -922,7 +925,7 @@ public class FSEditLogLoader {
* @throws IOException
*/
private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
- INodeFile file) throws IOException {
+ INodesInPath iip, INodeFile file) throws IOException {
// Update its block list
BlockInfo[] oldBlocks = file.getBlocks();
Block[] newBlocks = op.getBlocks();
@@ -976,7 +979,7 @@ public class FSEditLogLoader {
+ path);
}
Block oldBlock = oldBlocks[oldBlocks.length - 1];
- boolean removed = fsDir.unprotectedRemoveBlock(path, file, oldBlock);
+ boolean removed = fsDir.unprotectedRemoveBlock(path, iip, file, oldBlock);
if (!removed && !(op instanceof UpdateBlocksOp)) {
throw new IOException("Trying to delete non-existant block " + oldBlock);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index e26f052..0a92054 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -596,7 +596,7 @@ public class FSImageFormat {
// Rename .snapshot paths if we're doing an upgrade
parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion());
final INodeDirectory parent = INodeDirectory.valueOf(
- namesystem.dir.getNode(parentPath, true), parentPath);
+ namesystem.dir.getINode(parentPath, true), parentPath);
return loadChildren(parent, in, counter);
}
@@ -940,8 +940,8 @@ public class FSImageFormat {
inSnapshot = true;
} else {
path = renameReservedPathsOnUpgrade(path, getLayoutVersion());
- final INodesInPath iip = fsDir.getLastINodeInPath(path);
- oldnode = INodeFile.valueOf(iip.getINode(0), path);
+ final INodesInPath iip = fsDir.getINodesInPath(path, true);
+ oldnode = INodeFile.valueOf(iip.getLastINode(), path);
}
FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78e3a7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 5dd5920..b4b897a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2028,7 +2028,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (!createParent) {
dir.verifyParentDir(iip, link);
}
- if (!dir.isValidToCreate(link)) {
+ if (!dir.isValidToCreate(link, iip)) {
throw new IOException("failed to create link " + link
+" either because the filename is invalid or the file exists");
}
@@ -2039,7 +2039,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkFsObjectLimit();
// add symbolic link to namespace
- addSymlink(link, target, dirPerms, createParent, logRetryCache);
+ addSymlink(link, iip, target, dirPerms, createParent, logRetryCache);
resultingStat = getAuditFileInfo(link, false);
} finally {
writeUnlock();
@@ -2191,11 +2191,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
checkOperation(OperationCategory.READ);
filename = dir.resolvePath(pc, filename, pathComponents);
- final INodesInPath iip = dir.getINodesInPath(filename, true);
+ final INodesInPath iip = dir.getINodesInPath(filename, false);
if (isPermissionEnabled) {
dir.checkTraverse(pc, iip);
}
- return dir.getPreferredBlockSize(filename);
+ return INodeFile.valueOf(iip.getLastINode(), filename)
+ .getPreferredBlockSize();
} finally {
readUnlock();
}
@@ -2491,14 +2492,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (overwrite) {
toRemoveBlocks = new BlocksMapUpdateInfo();
List<INode> toRemoveINodes = new ChunkedArrayList<INode>();
- long ret = dir.delete(src, toRemoveBlocks, toRemoveINodes, now());
+ long ret = dir.delete(iip, toRemoveBlocks, toRemoveINodes, now());
if (ret >= 0) {
incrDeletedFileCount(ret);
removePathAndBlocks(src, null, toRemoveINodes, true);
}
} else {
// If lease soft limit time is expired, recover the lease
- recoverLeaseInternal(myFile, src, holder, clientMachine, false);
+ recoverLeaseInternal(iip, src, holder, clientMachine, false);
throw new FileAlreadyExistsException(src + " for client " +
clientMachine + " already exists");
}
@@ -2508,10 +2509,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
INodeFile newNode = null;
// Always do an implicit mkdirs for parent directory tree.
- Path parent = new Path(src).getParent();
- if (parent != null && FSDirMkdirOp.mkdirsRecursively(dir,
- parent.toString(), permissions, true, now())) {
- newNode = dir.addFile(src, permissions, replication, blockSize,
+ INodesInPath parentIIP = iip.getParentINodesInPath();
+ if (parentIIP != null && (parentIIP = FSDirMkdirOp.mkdirsRecursively(dir,
+ parentIIP, permissions, true, now())) != null) {
+ iip = INodesInPath.append(parentIIP, newNode, iip.getLastLocalName());
+ newNode = dir.addFile(iip, src, permissions, replication, blockSize,
holder, clientMachine);
}
@@ -2621,12 +2623,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
"Cannot append to lazy persist file " + src);
}
// Opening an existing file for write - may need to recover lease.
- recoverLeaseInternal(myFile, src, holder, clientMachine, false);
+ recoverLeaseInternal(iip, src, holder, clientMachine, false);
- // recoverLeaseInternal may create a new InodeFile via
- // finalizeINodeFileUnderConstruction so we need to refresh
- // the referenced file.
- myFile = INodeFile.valueOf(dir.getINode(src), src, true);
final BlockInfo lastBlock = myFile.getLastBlock();
// Check that the block has at least minimum replication.
if(lastBlock != null && lastBlock.isComplete() &&
@@ -2634,8 +2632,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new IOException("append: lastBlock=" + lastBlock +
" of src=" + src + " is not sufficiently replicated yet.");
}
- return prepareFileForWrite(src, myFile, holder, clientMachine, true,
- iip.getLatestSnapshotId(), logRetryCache);
+ return prepareFileForWrite(src, iip, holder, clientMachine, true,
+ logRetryCache);
} catch (IOException ie) {
NameNode.stateChangeLog.warn("DIR* NameSystem.append: " +ie.getMessage());
throw ie;
@@ -2643,11 +2641,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
- * Replace current node with a INodeUnderConstruction.
+ * Convert current node to under construction.
* Recreate in-memory lease record.
*
* @param src path to the file
- * @param file existing file object
* @param leaseHolder identifier of the lease holder on this file
* @param clientMachine identifier of the client machine
* @param writeToEditLog whether to persist this change to the edit log
@@ -2657,26 +2654,25 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @throws UnresolvedLinkException
* @throws IOException
*/
- LocatedBlock prepareFileForWrite(String src, INodeFile file,
- String leaseHolder, String clientMachine,
- boolean writeToEditLog,
- int latestSnapshot, boolean logRetryCache)
- throws IOException {
- file.recordModification(latestSnapshot);
- final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine);
+ LocatedBlock prepareFileForWrite(String src, INodesInPath iip,
+ String leaseHolder, String clientMachine, boolean writeToEditLog,
+ boolean logRetryCache) throws IOException {
+ final INodeFile file = iip.getLastINode().asFile();
+ file.recordModification(iip.getLatestSnapshotId());
+ file.toUnderConstruction(leaseHolder, clientMachine);
- leaseManager.addLease(cons.getFileUnderConstructionFeature()
- .getClientName(), src);
+ leaseManager.addLease(
+ file.getFileUnderConstructionFeature().getClientName(), src);
- LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons);
+ LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(file);
if (ret != null) {
// update the quota: use the preferred block size for UC block
final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
- dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication());
+ dir.updateSpaceConsumed(iip, 0, diff * file.getBlockReplication());
}
if (writeToEditLog) {
- getEditLog().logOpenFile(src, cons, false, logRetryCache);
+ getEditLog().logOpenFile(src, file, false, logRetryCache);
}
return ret;
}
@@ -2716,7 +2712,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
dir.checkPathAccess(pc, iip, FsAction.WRITE);
}
- recoverLeaseInternal(inode, src, holder, clientMachine, true);
+ recoverLeaseInternal(iip, src, holder, clientMachine, true);
} catch (StandbyException se) {
skipSync = true;
throw se;
@@ -2731,11 +2727,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return false;
}
- private void recoverLeaseInternal(INodeFile fileInode,
+ private void recoverLeaseInternal(INodesInPath iip,
String src, String holder, String clientMachine, boolean force)
throws IOException {
assert hasWriteLock();
- if (fileInode != null && fileInode.isUnderConstruction()) {
+ INodeFile file = iip.getLastINode().asFile();
+ if (file != null && file.isUnderConstruction()) {
//
// If the file is under construction , then it must be in our
// leases. Find the appropriate lease record.
@@ -2758,7 +2755,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
//
// Find the original holder.
//
- FileUnderConstructionFeature uc = fileInode.getFileUnderConstructionFeature();
+ FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
String clientName = uc.getClientName();
lease = leaseManager.getLease(clientName);
if (lease == null) {
@@ -2772,7 +2769,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// close only the file src
LOG.info("recoverLease: " + lease + ", src=" + src +
" from client " + clientName);
- internalReleaseLease(lease, src, holder);
+ internalReleaseLease(lease, src, iip, holder);
} else {
assert lease.getHolder().equals(clientName) :
"Current lease holder " + lease.getHolder() +
@@ -2784,13 +2781,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (lease.expiredSoftLimit()) {
LOG.info("startFile: recover " + lease + ", src=" + src + " client "
+ clientName);
- boolean isClosed = internalReleaseLease(lease, src, null);
+ boolean isClosed = internalReleaseLease(lease, src, iip, null);
if(!isClosed)
throw new RecoveryInProgressException(
"Failed to close file " + src +
". Lease recovery is in progress. Try again later.");
} else {
- final BlockInfo lastBlock = fileInode.getLastBlock();
+ final BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null
&& lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
throw new RecoveryInProgressException("Recovery in progress, file ["
@@ -2822,10 +2819,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
private LastBlockWithStatus appendFileInt(final String srcArg, String holder,
- String clientMachine, boolean logRetryCache)
- throws AccessControlException, SafeModeException,
- FileAlreadyExistsException, FileNotFoundException,
- ParentNotDirectoryException, IOException {
+ String clientMachine, boolean logRetryCache) throws IOException {
String src = srcArg;
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: src=" + src
@@ -2892,10 +2886,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
ExtendedBlock previous, Set<Node> excludedNodes,
- List<String> favoredNodes)
- throws LeaseExpiredException, NotReplicatedYetException,
- QuotaExceededException, SafeModeException, UnresolvedLinkException,
- IOException {
+ List<String> favoredNodes) throws IOException {
final long blockSize;
final int replication;
final byte storagePolicyID;
@@ -2983,7 +2974,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// commit the last block and complete it if it has minimum replicas
- commitOrCompleteLastBlock(pendingFile,
+ commitOrCompleteLastBlock(pendingFile, fileState.iip,
ExtendedBlock.getLocalBlock(previous));
// allocate new block, record block locations in INode.
@@ -3023,10 +3014,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
static class FileState {
public final INodeFile inode;
public final String path;
+ public final INodesInPath iip;
- public FileState(INodeFile inode, String fullPath) {
+ public FileState(INodeFile inode, String fullPath, INodesInPath iip) {
this.inode = inode;
this.path = fullPath;
+ this.iip = iip;
}
}
@@ -3046,18 +3039,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkFsObjectLimit();
Block previousBlock = ExtendedBlock.getLocalBlock(previous);
- INode inode;
+ final INode inode;
+ final INodesInPath iip;
if (fileId == INodeId.GRANDFATHER_INODE_ID) {
// Older clients may not have given us an inode ID to work with.
// In this case, we have to try to resolve the path and hope it
// hasn't changed or been deleted since the file was opened for write.
- final INodesInPath iip = dir.getINodesInPath4Write(src);
+ iip = dir.getINodesInPath4Write(src);
inode = iip.getLastINode();
} else {
// Newer clients pass the inode ID, so we can just get the inode
// directly.
inode = dir.getInode(fileId);
- if (inode != null) src = inode.getFullPathName();
+ iip = INodesInPath.fromINode(inode);
+ if (inode != null) {
+ src = iip.getPath();
+ }
}
final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
BlockInfo lastBlockInFile = pendingFile.getLastBlock();
@@ -3117,7 +3114,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
offset);
- return new FileState(pendingFile, src);
+ return new FileState(pendingFile, src, iip);
} else {
// Case 3
throw new IOException("Cannot allocate block in " + src + ": " +
@@ -3130,7 +3127,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (!checkFileProgress(src, pendingFile, false)) {
throw new NotReplicatedYetException("Not replicated yet: " + src);
}
- return new FileState(pendingFile, src);
+ return new FileState(pendingFile, src, iip);
}
LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
@@ -3208,8 +3205,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* The client would like to let go of the given block
*/
boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
- throws LeaseExpiredException, FileNotFoundException,
- UnresolvedLinkException, IOException {
+ throws IOException {
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: " + b
+ "of file " + src);
@@ -3225,21 +3221,24 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
src = dir.resolvePath(pc, src, pathComponents);
final INode inode;
+ final INodesInPath iip;
if (fileId == INodeId.GRANDFATHER_INODE_ID) {
// Older clients may not have given us an inode ID to work with.
// In this case, we have to try to resolve the path and hope it
// hasn't changed or been deleted since the file was opened for write.
- inode = dir.getINode(src);
+ iip = dir.getINodesInPath(src, true);
+ inode = iip.getLastINode();
} else {
inode = dir.getInode(fileId);
- if (inode != null) src = inode.getFullPathName();
+ iip = INodesInPath.fromINode(inode);
+ if (inode != null) {
+ src = iip.getPath();
+ }
}
final INodeFile file = checkLease(src, holder, inode, fileId);
- //
// Remove the block from the pending creates list
- //
- boolean removed = dir.removeBlock(src, file,
+ boolean removed = dir.removeBlock(src, iip, file,
ExtendedBlock.getLocalBlock(b));
if (!removed) {
return true;
@@ -3258,8 +3257,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
private INodeFile checkLease(String src, String holder, INode inode,
- long fileId)
- throws LeaseExpiredException, FileNotFoundException {
+ long fileId) throws LeaseExpiredException, FileNotFoundException {
assert hasReadLock();
final String ident = src + " (inode " + fileId + ")";
if (inode == null) {
@@ -3336,29 +3334,30 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return success;
}
- private boolean completeFileInternal(String src,
- String holder, Block last, long fileId) throws SafeModeException,
- UnresolvedLinkException, IOException {
+ private boolean completeFileInternal(String src, String holder, Block last,
+ long fileId) throws IOException {
assert hasWriteLock();
final INodeFile pendingFile;
+ final INodesInPath iip;
+ INode inode = null;
try {
- final INode inode;
if (fileId == INodeId.GRANDFATHER_INODE_ID) {
// Older clients may not have given us an inode ID to work with.
// In this case, we have to try to resolve the path and hope it
// hasn't changed or been deleted since the file was opened for write.
- final INodesInPath iip = dir.getLastINodeInPath(src);
- inode = iip.getINode(0);
+ iip = dir.getINodesInPath(src, true);
+ inode = iip.getLastINode();
} else {
inode = dir.getInode(fileId);
- if (inode != null) src = inode.getFullPathName();
+ iip = INodesInPath.fromINode(inode);
+ if (inode != null) {
+ src = iip.getPath();
+ }
}
pendingFile = checkLease(src, holder, inode, fileId);
} catch (LeaseExpiredException lee) {
- final INode inode = dir.getINode(src);
- if (inode != null
- && inode.isFile()
- && !inode.asFile().isUnderConstruction()) {
+ if (inode != null && inode.isFile() &&
+ !inode.asFile().isUnderConstruction()) {
// This could be a retry RPC - i.e the client tried to close
// the file, but missed the RPC response. Thus, it is trying
// again to close the file. If the file still exists and
@@ -3383,7 +3382,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// commit the last block and complete it if it has minimum replicas
- commitOrCompleteLastBlock(pendingFile, last);
+ commitOrCompleteLastBlock(pendingFile, iip, last);
if (!checkFileProgress(src, pendingFile, true)) {
return false;
@@ -3618,7 +3617,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
long mtime = now();
// Unlink the target directory from directory tree
- long filesRemoved = dir.delete(src, collectedBlocks, removedINodes,
+ long filesRemoved = dir.delete(iip, collectedBlocks, removedINodes,
mtime);
if (filesRemoved < 0) {
return false;
@@ -3885,7 +3884,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @throws IOException if path does not exist
*/
void fsync(String src, long fileId, String clientName, long lastBlockLength)
- throws IOException, UnresolvedLinkException {
+ throws IOException {
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
@@ -3933,15 +3932,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* false if block recovery has been initiated. Since the lease owner
* has been changed and logged, caller should call logSync().
*/
- boolean internalReleaseLease(Lease lease, String src,
- String recoveryLeaseHolder) throws AlreadyBeingCreatedException,
- IOException, UnresolvedLinkException {
+ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip,
+ String recoveryLeaseHolder) throws IOException {
LOG.info("Recovering " + lease + ", src=" + src);
assert !isInSafeMode();
assert hasWriteLock();
- final INodesInPath iip = dir.getLastINodeInPath(src);
- final INodeFile pendingFile = iip.getINode(0).asFile();
+ final INodeFile pendingFile = iip.getLastINode().asFile();
int nrBlocks = pendingFile.numBlocks();
BlockInfo[] blocks = pendingFile.getBlocks();
@@ -4070,7 +4067,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
private void commitOrCompleteLastBlock(final INodeFile fileINode,
- final Block commitBlock) throws IOException {
+ final INodesInPath iip, final Block commitBlock) throws IOException {
assert hasWriteLock();
Preconditions.checkArgument(fileINode.isUnderConstruction());
if (!blockManager.commitOrCompleteLastBlock(fileINode, commitBlock)) {
@@ -4081,8 +4078,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
if (diff > 0) {
try {
- String path = fileINode.getFullPathName();
- dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
+ dir.updateSpaceConsumed(iip, 0, -diff*fileINode.getFileReplication());
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
}
@@ -4090,8 +4086,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
private void finalizeINodeFileUnderConstruction(String src,
- INodeFile pendingFile, int latestSnapshot) throws IOException,
- UnresolvedLinkException {
+ INodeFile pendingFile, int latestSnapshot) throws IOException {
assert hasWriteLock();
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
@@ -4103,13 +4098,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// The file is no longer pending.
// Create permanent INode, update blocks. No need to replace the inode here
// since we just remove the uc feature from pendingFile
- final INodeFile newFile = pendingFile.toCompleteFile(now());
+ pendingFile.toCompleteFile(now());
waitForLoadingFSImage();
// close file and persist block allocations for this file
- closeFile(src, newFile);
+ closeFile(src, pendingFile);
- blockManager.checkReplication(newFile);
+ blockManager.checkReplication(pendingFile);
}
@VisibleForTesting
@@ -4126,11 +4121,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return false;
}
- INodeFile inodeUC = (INodeFile) bc;
- String fullName = inodeUC.getName();
+ String fullName = bc.getName();
try {
if (fullName != null && fullName.startsWith(Path.SEPARATOR)
- && dir.getINode(fullName) == inodeUC) {
+ && dir.getINode(fullName) == bc) {
// If file exists in normal path then no need to look in snapshot
return false;
}
@@ -4139,7 +4133,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return false;
}
/*
- * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and
+ * 1. if bc is under construction and also with snapshot, and
* bc is not in the current fsdirectory tree, bc must represent a snapshot
* file.
* 2. if fullName is not an absolute path, bc cannot be existent in the
@@ -4153,8 +4147,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void commitBlockSynchronization(ExtendedBlock lastblock,
long newgenerationstamp, long newlength,
boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,
- String[] newtargetstorages)
- throws IOException, UnresolvedLinkException {
+ String[] newtargetstorages) throws IOException {
LOG.info("commitBlockSynchronization(lastblock=" + lastblock
+ ", newgenerationstamp=" + newgenerationstamp
+ ", newlength=" + newlength
@@ -4312,10 +4305,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@VisibleForTesting
String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
throws IOException {
- String src = pendingFile.getFullPathName();
+ final INodesInPath iip = INodesInPath.fromINode(pendingFile);
+ final String src = iip.getPath();
// commit the last block and complete it if it has minimum replicas
- commitOrCompleteLastBlock(pendingFile, storedBlock);
+ commitOrCompleteLastBlock(pendingFile, iip, storedBlock);
//remove lease, close file
finalizeINodeFileUnderConstruction(src, pendingFile,
@@ -4515,7 +4509,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Add the given symbolic link to the fs. Record it in the edits log.
*/
- private INodeSymlink addSymlink(String path, String target,
+ private INodeSymlink addSymlink(String path, INodesInPath iip, String target,
PermissionStatus dirPerms,
boolean createParent, boolean logRetryCache)
throws UnresolvedLinkException, FileAlreadyExistsException,
@@ -4524,15 +4518,17 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final long modTime = now();
if (createParent) {
- final String parent = new Path(path).getParent().toString();
- if (!FSDirMkdirOp.mkdirsRecursively(dir, parent, dirPerms, true,
- modTime)) {
+ INodesInPath parentIIP = iip.getParentINodesInPath();
+ if (parentIIP == null || (parentIIP = FSDirMkdirOp.mkdirsRecursively(dir,
+ parentIIP, dirPerms, true, modTime)) == null) {
return null;
+ } else {
+ iip = INodesInPath.append(parentIIP, null, iip.getLastLocalName());
}
}
final String userName = dirPerms.getUserName();
long id = dir.allocateNewInodeId();
- INodeSymlink newNode = dir.addSymlink(id, path, target, modTime, modTime,
+ INodeSymlink newNode = dir.addSymlink(iip, id, target, modTime, modTime,
new PermissionStatus(userName, null, FsPermission.getDefault()));
if (newNode == null) {
NameNode.stateChangeLog.info("addSymlink: failed to add " + path);