You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2012/12/03 19:05:06 UTC
svn commit: r1416603 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/
src/main/native/ s...
Author: suresh
Date: Mon Dec 3 18:04:51 2012
New Revision: 1416603
URL: http://svn.apache.org/viewvc?rev=1416603&view=rev
Log:
Merging trunk to HDFS-2802 branch.
Added:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
- copied unchanged from r1416602, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1415804-1416602
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Dec 3 18:04:51 2012
@@ -176,6 +176,9 @@ Trunk (Unreleased)
HDFS-4209. Clean up the addNode/addChild/addChildNoQuotaCheck methods in
FSDirectory and INodeDirectory. (szetszwo)
+ HDFS-3358. Specify explicitly that the NN UI status total is talking
+ of persistent objects on heap. (harsh)
+
OPTIMIZATIONS
BUG FIXES
@@ -646,6 +649,8 @@ Release 2.0.3-alpha - Unreleased
of it is undefined after the iteration or modifications of the map.
(szetszwo)
+ HDFS-4231. BackupNode: Introduce BackupState. (shv)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -2035,6 +2040,11 @@ Release 0.23.6 - UNRELEASED
BUG FIXES
+ HDFS-4247. saveNamespace should be tolerant of dangling lease (daryn)
+
+ HDFS-4248. Renaming directories may incorrectly remove the paths in leases
+ under the tree. (daryn via szetszwo)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1415804-1416602
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Mon Dec 3 18:04:51 2012
@@ -24,6 +24,7 @@ import java.net.SocketTimeoutException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.NameNodeProxies;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
@@ -414,14 +416,23 @@ public class BackupNode extends NameNode
+ HdfsConstants.LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();
return nsInfo;
}
-
+
@Override
+ protected String getNameServiceId(Configuration conf) {
+ return DFSUtil.getBackupNameServiceId(conf);
+ }
+
+ protected HAState createHAState() {
+ return new BackupState();
+ }
+
+ @Override // NameNode
protected NameNodeHAContext createHAContext() {
return new BNHAContext();
}
-
+
private class BNHAContext extends NameNodeHAContext {
- @Override // NameNode
+ @Override // NameNodeHAContext
public void checkOperation(OperationCategory op)
throws StandbyException {
if (op == OperationCategory.UNCHECKED ||
@@ -435,10 +446,42 @@ public class BackupNode extends NameNode
throw new StandbyException(msg);
}
}
- }
-
- @Override
- protected String getNameServiceId(Configuration conf) {
- return DFSUtil.getBackupNameServiceId(conf);
+
+ @Override // NameNodeHAContext
+ public void prepareToStopStandbyServices() throws ServiceFailedException {
+ }
+
+ /**
+ * Start services for BackupNode.
+ * <p>
+ * The following services should be muted
+ * (not run or not pass any control commands to DataNodes)
+ * on BackupNode:
+ * {@link LeaseManager.Monitor} protected by SafeMode.
+ * {@link BlockManager.ReplicationMonitor} protected by SafeMode.
+ * {@link HeartbeatManager.Monitor} protected by SafeMode.
+ * {@link DecommissionManager.Monitor} need to prohibit refreshNodes().
+ * {@link PendingReplicationBlocks.PendingReplicationMonitor} harmless,
+ * because ReplicationMonitor is muted.
+ */
+ @Override
+ public void startActiveServices() throws IOException {
+ try {
+ namesystem.startActiveServices();
+ } catch (Throwable t) {
+ doImmediateShutdown(t);
+ }
+ }
+
+ @Override
+ public void stopActiveServices() throws IOException {
+ try {
+ if (namesystem != null) {
+ namesystem.stopActiveServices();
+ }
+ } catch (Throwable t) {
+ doImmediateShutdown(t);
+ }
+ }
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Mon Dec 3 18:04:51 2012
@@ -383,7 +383,7 @@ public class FSDirectory implements Clos
writeLock();
try {
// file is closed
- file.setModificationTimeForce(now);
+ file.setModificationTime(now);
fsImage.getEditLog().logCloseFile(path, file);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.closeFile: "
@@ -585,8 +585,10 @@ public class FSDirectory implements Clos
+ src + " is renamed to " + dst);
}
// update modification time of dst and the parent of src
- srcInodes[srcInodes.length-2].setModificationTime(timestamp);
- dstInodes[dstInodes.length-2].setModificationTime(timestamp);
+ srcInodes[srcInodes.length-2].updateModificationTime(timestamp);
+ dstInodes[dstInodes.length-2].updateModificationTime(timestamp);
+ // update moved leases with new filename
+ getFSNamesystem().unprotectedChangeLease(src, dst);
return true;
}
} finally {
@@ -750,8 +752,10 @@ public class FSDirectory implements Clos
"DIR* FSDirectory.unprotectedRenameTo: " + src
+ " is renamed to " + dst);
}
- srcInodes[srcInodes.length - 2].setModificationTime(timestamp);
- dstInodes[dstInodes.length - 2].setModificationTime(timestamp);
+ srcInodes[srcInodes.length - 2].updateModificationTime(timestamp);
+ dstInodes[dstInodes.length - 2].updateModificationTime(timestamp);
+ // update moved lease with new filename
+ getFSNamesystem().unprotectedChangeLease(src, dst);
// Collect the blocks and remove the lease for previous dst
int filesDeleted = 0;
@@ -986,12 +990,12 @@ public class FSDirectory implements Clos
if(nodeToRemove == null) continue;
nodeToRemove.setBlocks(null);
- trgParent.removeChild(nodeToRemove);
+ trgParent.removeChild(nodeToRemove, trgINodesInPath.getLatestSnapshot());
count++;
}
- trgInode.setModificationTimeForce(timestamp);
- trgParent.setModificationTime(timestamp);
+ trgInode.setModificationTime(timestamp);
+ trgParent.updateModificationTime(timestamp);
// update quota on the parent directory ('count' files removed, 0 space)
unprotectedUpdateCount(trgINodesInPath, trgINodes.length-1, -count, 0);
}
@@ -1129,7 +1133,7 @@ public class FSDirectory implements Clos
return 0;
}
// set the parent's modification time
- inodes[inodes.length - 2].setModificationTime(mtime);
+ inodes[inodes.length - 2].updateModificationTime(mtime);
int filesRemoved = targetNode.collectSubtreeBlocksAndClear(collectedBlocks);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
@@ -1165,10 +1169,10 @@ public class FSDirectory implements Clos
/**
* Replaces the specified INode.
*/
- private void replaceINodeUnsynced(String path, INode oldnode, INode newnode
- ) throws IOException {
+ private void replaceINodeUnsynced(String path, INode oldnode, INode newnode,
+ Snapshot latestSnapshot) throws IOException {
//remove the old node from the namespace
- if (!oldnode.removeNode()) {
+ if (!oldnode.removeNode(latestSnapshot)) {
final String mess = "FSDirectory.replaceINodeUnsynced: failed to remove "
+ path;
NameNode.stateChangeLog.warn("DIR* " + mess);
@@ -1183,10 +1187,10 @@ public class FSDirectory implements Clos
* Replaces the specified INodeDirectory.
*/
public void replaceINodeDirectory(String path, INodeDirectory oldnode,
- INodeDirectory newnode) throws IOException {
+ INodeDirectory newnode, Snapshot latestSnapshot) throws IOException {
writeLock();
try {
- replaceINodeUnsynced(path, oldnode, newnode);
+ replaceINodeUnsynced(path, oldnode, newnode, latestSnapshot);
//update children's parent directory
for(INode i : newnode.getChildrenList(null)) {
@@ -1200,11 +1204,11 @@ public class FSDirectory implements Clos
/**
* Replaces the specified INodeFile with the specified one.
*/
- public void replaceNode(String path, INodeFile oldnode, INodeFile newnode
- ) throws IOException {
+ public void replaceNode(String path, INodeFile oldnode, INodeFile newnode,
+ Snapshot latestSnapshot) throws IOException {
writeLock();
try {
- replaceINodeUnsynced(path, oldnode, newnode);
+ replaceINodeUnsynced(path, oldnode, newnode, latestSnapshot);
//Currently, oldnode and newnode are assumed to contain the same blocks.
//Otherwise, blocks need to be removed from the blocksMap.
@@ -1273,13 +1277,9 @@ public class FSDirectory implements Clos
String srcs = normalizePath(src);
readLock();
try {
- INode targetNode = rootDir.getNode(srcs, resolveLink);
- if (targetNode == null) {
- return null;
- }
- else {
- return createFileStatus(HdfsFileStatus.EMPTY_NAME, targetNode);
- }
+ final INodesInPath inodesInPath = rootDir.getINodesInPath(srcs, resolveLink);
+ final INode i = inodesInPath.getINode(0);
+ return i == null? null: createFileStatus(HdfsFileStatus.EMPTY_NAME, i);
} finally {
readUnlock();
}
@@ -1303,9 +1303,16 @@ public class FSDirectory implements Clos
* Get {@link INode} associated with the file / directory.
*/
public INode getINode(String src) throws UnresolvedLinkException {
+ return getINodesInPath(src).getINode(0);
+ }
+
+ /**
+ * Get {@link INode} associated with the file / directory.
+ */
+ public INodesInPath getINodesInPath(String src) throws UnresolvedLinkException {
readLock();
try {
- return rootDir.getNode(src, true);
+ return rootDir.getINodesInPath(src, true);
} finally {
readUnlock();
}
@@ -1800,7 +1807,8 @@ public class FSDirectory implements Clos
if (inodes[pos-1] == null) {
throw new NullPointerException("Panic: parent does not exist");
}
- final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true);
+ final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true,
+ inodesInPath.getLatestSnapshot());
if (!added) {
updateCount(inodesInPath, pos, -counts.getNsCount(), -counts.getDsCount(), true);
}
@@ -1824,7 +1832,8 @@ public class FSDirectory implements Clos
private INode removeLastINode(final INodesInPath inodesInPath) {
final INode[] inodes = inodesInPath.getINodes();
final int pos = inodes.length - 1;
- INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos]);
+ INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos],
+ inodesInPath.getLatestSnapshot());
if (removedNode != null) {
INode.DirCounts counts = new INode.DirCounts();
removedNode.spaceConsumedInTree(counts);
@@ -1965,8 +1974,8 @@ public class FSDirectory implements Clos
}
String srcs = normalizePath(src);
- final INode[] inodes = rootDir.getMutableINodesInPath(srcs, true)
- .getINodes();
+ final INodesInPath inodesInPath = rootDir.getMutableINodesInPath(srcs, true);
+ final INode[] inodes = inodesInPath.getINodes();
INodeDirectory dirNode = INodeDirectory.valueOf(inodes[inodes.length-1], srcs);
if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
throw new IllegalArgumentException("Cannot clear namespace quota on root.");
@@ -1988,7 +1997,7 @@ public class FSDirectory implements Clos
INodeDirectory newNode = new INodeDirectory(dirNode);
INodeDirectory parent = (INodeDirectory)inodes[inodes.length-2];
dirNode = newNode;
- parent.replaceChild(newNode);
+ parent.replaceChild(newNode, inodesInPath.getLatestSnapshot());
}
} else {
// a non-quota directory; so replace it with a directory with quota
@@ -1997,7 +2006,7 @@ public class FSDirectory implements Clos
// non-root directory node; parent != null
INodeDirectory parent = (INodeDirectory)inodes[inodes.length-2];
dirNode = newNode;
- parent.replaceChild(newNode);
+ parent.replaceChild(newNode, inodesInPath.getLatestSnapshot());
}
return (oldNsQuota != nsQuota || oldDsQuota != dsQuota) ? dirNode : null;
}
@@ -2061,7 +2070,7 @@ public class FSDirectory implements Clos
assert hasWriteLock();
boolean status = false;
if (mtime != -1) {
- inode.setModificationTimeForce(mtime);
+ inode.setModificationTime(mtime);
status = true;
}
if (atime != -1) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Mon Dec 3 18:04:51 2012
@@ -31,7 +31,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
@@ -58,6 +57,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.util.Holder;
@@ -246,7 +246,8 @@ public class FSEditLogLoader {
// 3. OP_ADD to open file for append
// See if the file already exists (persistBlocks call)
- INodeFile oldFile = getINodeFile(fsDir, addCloseOp.path);
+ final INodesInPath inodesInPath = fsDir.getINodesInPath(addCloseOp.path);
+ INodeFile oldFile = toINodeFile(inodesInPath.getINode(0), addCloseOp.path);
INodeFile newFile = oldFile;
if (oldFile == null) { // this is OP_ADD on a new file (case 1)
// versions > 0 support per file replication
@@ -272,7 +273,7 @@ public class FSEditLogLoader {
}
fsNamesys.prepareFileForWrite(addCloseOp.path, oldFile,
addCloseOp.clientName, addCloseOp.clientMachine, null,
- false);
+ false, inodesInPath.getLatestSnapshot());
newFile = getINodeFile(fsDir, addCloseOp.path);
}
}
@@ -282,7 +283,7 @@ public class FSEditLogLoader {
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime);
- newFile.setModificationTimeForce(addCloseOp.mtime);
+ newFile.setModificationTime(addCloseOp.mtime);
updateBlocks(fsDir, addCloseOp, newFile);
break;
}
@@ -296,7 +297,8 @@ public class FSEditLogLoader {
" clientMachine " + addCloseOp.clientMachine);
}
- INodeFile oldFile = getINodeFile(fsDir, addCloseOp.path);
+ final INodesInPath inodesInPath = fsDir.getINodesInPath(addCloseOp.path);
+ INodeFile oldFile = toINodeFile(inodesInPath.getINode(0), addCloseOp.path);
if (oldFile == null) {
throw new IOException("Operation trying to close non-existent file " +
addCloseOp.path);
@@ -304,7 +306,7 @@ public class FSEditLogLoader {
// Update the salient file attributes.
oldFile.setAccessTime(addCloseOp.atime);
- oldFile.setModificationTimeForce(addCloseOp.mtime);
+ oldFile.setModificationTime(addCloseOp.mtime);
updateBlocks(fsDir, addCloseOp, oldFile);
// Now close the file
@@ -322,7 +324,8 @@ public class FSEditLogLoader {
INodeFileUnderConstruction ucFile = (INodeFileUnderConstruction) oldFile;
fsNamesys.leaseManager.removeLeaseWithPrefixPath(addCloseOp.path);
INodeFile newFile = ucFile.convertToInodeFile();
- fsDir.replaceNode(addCloseOp.path, ucFile, newFile);
+ fsDir.replaceNode(addCloseOp.path, ucFile, newFile,
+ inodesInPath.getLatestSnapshot());
}
break;
}
@@ -360,10 +363,8 @@ public class FSEditLogLoader {
}
case OP_RENAME_OLD: {
RenameOldOp renameOp = (RenameOldOp)op;
- HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false);
fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst,
renameOp.timestamp);
- fsNamesys.unprotectedChangeLease(renameOp.src, renameOp.dst, dinfo);
break;
}
case OP_DELETE: {
@@ -433,11 +434,8 @@ public class FSEditLogLoader {
}
case OP_RENAME: {
RenameOp renameOp = (RenameOp)op;
-
- HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false);
fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst,
renameOp.timestamp, renameOp.options);
- fsNamesys.unprotectedChangeLease(renameOp.src, renameOp.dst, dinfo);
break;
}
case OP_GET_DELEGATION_TOKEN: {
@@ -512,7 +510,11 @@ public class FSEditLogLoader {
private static INodeFile getINodeFile(FSDirectory fsDir, String path)
throws IOException {
- INode inode = fsDir.getINode(path);
+ return toINodeFile(fsDir.getINode(path), path);
+ }
+
+ private static INodeFile toINodeFile(INode inode, String path)
+ throws IOException {
if (inode != null) {
if (!(inode instanceof INodeFile)) {
throw new IOException("Operation trying to get non-file " + path);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Mon Dec 3 18:04:51 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
@@ -202,7 +203,7 @@ class FSImageFormat {
if (nsQuota != -1 || dsQuota != -1) {
fsDir.rootDir.setQuota(nsQuota, dsQuota);
}
- fsDir.rootDir.setModificationTime(root.getModificationTime());
+ fsDir.rootDir.cloneModificationTime(root);
fsDir.rootDir.clonePermissionStatus(root);
}
@@ -305,7 +306,7 @@ class FSImageFormat {
*/
void addToParent(INodeDirectory parent, INode child) {
// NOTE: This does not update space counts for parents
- if (!parent.addChild(child, false)) {
+ if (!parent.addChild(child, false, null)) {
return;
}
namesystem.dir.cacheName(child);
@@ -388,8 +389,9 @@ class FSImageFormat {
// verify that file exists in namespace
String path = cons.getLocalName();
- INodeFile oldnode = INodeFile.valueOf(fsDir.getINode(path), path);
- fsDir.replaceNode(path, oldnode, cons);
+ final INodesInPath inodesInPath = fsDir.getINodesInPath(path);
+ INodeFile oldnode = INodeFile.valueOf(inodesInPath.getINode(0), path);
+ fsDir.replaceNode(path, oldnode, cons, inodesInPath.getLatestSnapshot());
namesystem.leaseManager.addLease(cons.getClientName(), path);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Dec 3 18:04:51 2012
@@ -121,6 +121,7 @@ import org.apache.hadoop.fs.UnresolvedLi
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
@@ -165,7 +166,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
@@ -173,7 +173,9 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithLink;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
@@ -1036,7 +1038,8 @@ public class FSNamesystem implements Nam
long totalInodes = this.dir.totalInodes();
long totalBlocks = this.getBlocksTotal();
out.println(totalInodes + " files and directories, " + totalBlocks
- + " blocks = " + (totalInodes + totalBlocks) + " total");
+ + " blocks = " + (totalInodes + totalBlocks)
+ + " total filesystem objects");
blockManager.metaSave(out);
}
@@ -1820,7 +1823,9 @@ public class FSNamesystem implements Nam
try {
blockManager.verifyReplication(src, replication, clientMachine);
boolean create = flag.contains(CreateFlag.CREATE);
- final INode myFile = dir.getINode(src);
+
+ final INodesInPath inodesInPath = dir.getINodesInPath(src);
+ final INode myFile = inodesInPath.getINode(0);
if (myFile == null) {
if (!create) {
throw new FileNotFoundException("failed to overwrite or append to non-existent file "
@@ -1847,8 +1852,8 @@ public class FSNamesystem implements Nam
if (append && myFile != null) {
final INodeFile f = INodeFile.valueOf(myFile, src);
- return prepareFileForWrite(
- src, f, holder, clientMachine, clientNode, true);
+ return prepareFileForWrite(src, f, holder, clientMachine, clientNode,
+ true, inodesInPath.getLatestSnapshot());
} else {
// Now we can add the name to the filesystem. This file has no
// blocks associated with it.
@@ -1896,7 +1901,7 @@ public class FSNamesystem implements Nam
*/
LocatedBlock prepareFileForWrite(String src, INodeFile file,
String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
- boolean writeToEditLog) throws IOException {
+ boolean writeToEditLog, Snapshot latestSnapshot) throws IOException {
//TODO SNAPSHOT: INodeFileUnderConstruction with link
INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
file.getLocalNameBytes(),
@@ -1908,7 +1913,7 @@ public class FSNamesystem implements Nam
leaseHolder,
clientMachine,
clientNode);
- dir.replaceNode(src, file, cons);
+ dir.replaceNode(src, file, cons, latestSnapshot);
leaseManager.addLease(cons.getClientName(), src);
LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons);
@@ -2158,7 +2163,8 @@ public class FSNamesystem implements Nam
// have we exceeded the configured limit of fs objects.
checkFsObjectLimit();
- INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
+ final INodeFileUnderConstruction pendingFile = checkLease(
+ src, clientName, dir.getINode(src));
BlockInfo lastBlockInFile = pendingFile.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
@@ -2294,7 +2300,8 @@ public class FSNamesystem implements Nam
}
//check lease
- final INodeFileUnderConstruction file = checkLease(src, clientName);
+ final INodeFileUnderConstruction file = checkLease(
+ src, clientName, dir.getINode(src));
clientnode = file.getClientNode();
preferredblocksize = file.getPreferredBlockSize();
@@ -2340,7 +2347,9 @@ public class FSNamesystem implements Nam
throw new SafeModeException("Cannot abandon block " + b +
" for fle" + src, safeMode);
}
- INodeFileUnderConstruction file = checkLease(src, holder);
+ final INodesInPath inodesInPath = checkLease(src, holder);
+ final INodeFileUnderConstruction file
+ = (INodeFileUnderConstruction)inodesInPath.getINode(0);
dir.removeBlock(src, file, ExtendedBlock.getLocalBlock(b));
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: "
@@ -2357,11 +2366,13 @@ public class FSNamesystem implements Nam
return true;
}
- // make sure that we still have the lease on this file.
- private INodeFileUnderConstruction checkLease(String src, String holder)
+ /** make sure that we still have the lease on this file. */
+ private INodesInPath checkLease(String src, String holder)
throws LeaseExpiredException, UnresolvedLinkException {
assert hasReadOrWriteLock();
- return checkLease(src, holder, dir.getINode(src));
+ final INodesInPath inodesInPath = dir.getINodesInPath(src);
+ checkLease(src, holder, inodesInPath.getINode(0));
+ return inodesInPath;
}
private INodeFileUnderConstruction checkLease(String src, String holder,
@@ -2424,9 +2435,11 @@ public class FSNamesystem implements Nam
throw new SafeModeException("Cannot complete file " + src, safeMode);
}
- INodeFileUnderConstruction pendingFile;
+ final INodesInPath inodesInPath;
+ final INodeFileUnderConstruction pendingFile;
try {
- pendingFile = checkLease(src, holder);
+ inodesInPath = checkLease(src, holder);
+ pendingFile = (INodeFileUnderConstruction)inodesInPath.getINode(0);
} catch (LeaseExpiredException lee) {
final INode inode = dir.getINode(src);
if (inode != null && inode instanceof INodeFile && !inode.isUnderConstruction()) {
@@ -2454,7 +2467,8 @@ public class FSNamesystem implements Nam
return false;
}
- finalizeINodeFileUnderConstruction(src, pendingFile);
+ finalizeINodeFileUnderConstruction(src, pendingFile,
+ inodesInPath.getLatestSnapshot());
NameNode.stateChangeLog.info("DIR* completeFile: " + src + " is closed by "
+ holder);
@@ -2595,15 +2609,15 @@ public class FSNamesystem implements Nam
if (isPermissionEnabled) {
//We should not be doing this. This is move() not renameTo().
//but for now,
+ //NOTE: yes, this is bad! it's assuming much lower level behavior
+ // of rewriting the dst
String actualdst = dir.isDir(dst)?
dst + Path.SEPARATOR + new Path(src).getName(): dst;
checkParentAccess(src, FsAction.WRITE);
checkAncestorAccess(actualdst, FsAction.WRITE);
}
- HdfsFileStatus dinfo = dir.getFileInfo(dst, false);
if (dir.renameTo(src, dst)) {
- unprotectedChangeLease(src, dst, dinfo); // update lease with new filename
return true;
}
return false;
@@ -2654,9 +2668,7 @@ public class FSNamesystem implements Nam
checkAncestorAccess(dst, FsAction.WRITE);
}
- HdfsFileStatus dinfo = dir.getFileInfo(dst, false);
dir.renameTo(src, dst, options);
- unprotectedChangeLease(src, dst, dinfo); // update lease with new filename
}
/**
@@ -3036,7 +3048,8 @@ public class FSNamesystem implements Nam
if (isInSafeMode()) {
throw new SafeModeException("Cannot fsync file " + src, safeMode);
}
- INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
+ final INodeFileUnderConstruction pendingFile = checkLease(
+ src, clientName, dir.getINode(src));
if (lastBlockLength > 0) {
pendingFile.updateLengthOfLastBlock(lastBlockLength);
}
@@ -3068,8 +3081,9 @@ public class FSNamesystem implements Nam
assert !isInSafeMode();
assert hasWriteLock();
+ final INodesInPath inodesInPath = dir.getINodesInPath(src);
final INodeFileUnderConstruction pendingFile
- = INodeFileUnderConstruction.valueOf(dir.getINode(src), src);
+ = INodeFileUnderConstruction.valueOf(inodesInPath.getINode(0), src);
int nrBlocks = pendingFile.numBlocks();
BlockInfo[] blocks = pendingFile.getBlocks();
@@ -3086,7 +3100,8 @@ public class FSNamesystem implements Nam
// If there are no incomplete blocks associated with this file,
// then reap lease immediately and close the file.
if(nrCompleteBlocks == nrBlocks) {
- finalizeINodeFileUnderConstruction(src, pendingFile);
+ finalizeINodeFileUnderConstruction(src, pendingFile,
+ inodesInPath.getLatestSnapshot());
NameNode.stateChangeLog.warn("BLOCK*"
+ " internalReleaseLease: All existing blocks are COMPLETE,"
+ " lease removed, file closed.");
@@ -3134,7 +3149,8 @@ public class FSNamesystem implements Nam
// Close file if committed blocks are minimally replicated
if(penultimateBlockMinReplication &&
blockManager.checkMinReplication(lastBlock)) {
- finalizeINodeFileUnderConstruction(src, pendingFile);
+ finalizeINodeFileUnderConstruction(src, pendingFile,
+ inodesInPath.getLatestSnapshot());
NameNode.stateChangeLog.warn("BLOCK*"
+ " internalReleaseLease: Committed blocks are minimally replicated,"
+ " lease removed, file closed.");
@@ -3212,7 +3228,7 @@ public class FSNamesystem implements Nam
}
private void finalizeINodeFileUnderConstruction(String src,
- INodeFileUnderConstruction pendingFile)
+ INodeFileUnderConstruction pendingFile, Snapshot latestSnapshot)
throws IOException, UnresolvedLinkException {
assert hasWriteLock();
leaseManager.removeLease(pendingFile.getClientName(), src);
@@ -3220,7 +3236,7 @@ public class FSNamesystem implements Nam
// The file is no longer pending.
// Create permanent INode, update blocks
INodeFile newFile = pendingFile.convertToInodeFile();
- dir.replaceNode(src, pendingFile, newFile);
+ dir.replaceNode(src, pendingFile, newFile, latestSnapshot);
// close file and persist block allocations for this file
dir.closeFile(src, newFile);
@@ -3312,7 +3328,8 @@ public class FSNamesystem implements Nam
commitOrCompleteLastBlock(pendingFile, storedBlock);
//remove lease, close file
- finalizeINodeFileUnderConstruction(src, pendingFile);
+ finalizeINodeFileUnderConstruction(src, pendingFile,
+ INodeDirectorySnapshottable.findLatestSnapshot(pendingFile));
} else {
// If this commit does not want to close the file, persist blocks
dir.persistBlocks(src, pendingFile);
@@ -3483,9 +3500,9 @@ public class FSNamesystem implements Nam
private NNHAStatusHeartbeat createHaStatusHeartbeat() {
HAState state = haContext.getState();
NNHAStatusHeartbeat.State hbState;
- if (state instanceof ActiveState) {
+ if (state.getServiceState() == HAServiceState.ACTIVE) {
hbState = NNHAStatusHeartbeat.State.ACTIVE;
- } else if (state instanceof StandbyState) {
+ } else if (state.getServiceState() == HAServiceState.STANDBY) {
hbState = NNHAStatusHeartbeat.State.STANDBY;
} else {
throw new AssertionError("Invalid state: " + state.getClass());
@@ -4939,31 +4956,9 @@ public class FSNamesystem implements Nam
// rename was successful. If any part of the renamed subtree had
// files that were being written to, update with new filename.
- void unprotectedChangeLease(String src, String dst, HdfsFileStatus dinfo) {
- String overwrite;
- String replaceBy;
+ void unprotectedChangeLease(String src, String dst) {
assert hasWriteLock();
-
- boolean destinationExisted = true;
- if (dinfo == null) {
- destinationExisted = false;
- }
-
- if (destinationExisted && dinfo.isDir()) {
- Path spath = new Path(src);
- Path parent = spath.getParent();
- if (parent.isRoot()) {
- overwrite = parent.toString();
- } else {
- overwrite = parent.toString() + Path.SEPARATOR;
- }
- replaceBy = dst + Path.SEPARATOR;
- } else {
- overwrite = src;
- replaceBy = dst;
- }
-
- leaseManager.changeLease(src, dst, overwrite, replaceBy);
+ leaseManager.changeLease(src, dst);
}
/**
@@ -4974,19 +4969,13 @@ public class FSNamesystem implements Nam
// lock on our behalf. If we took the read lock here, we could block
// for fairness if a writer is waiting on the lock.
synchronized (leaseManager) {
- out.writeInt(leaseManager.countPath()); // write the size
-
- for (Lease lease : leaseManager.getSortedLeases()) {
- for(String path : lease.getPaths()) {
- // verify that path exists in namespace
- final INodeFileUnderConstruction cons;
- try {
- cons = INodeFileUnderConstruction.valueOf(dir.getINode(path), path);
- } catch (UnresolvedLinkException e) {
- throw new AssertionError("Lease files should reside on this FS");
- }
- FSImageSerialization.writeINodeUnderConstruction(out, cons, path);
- }
+ Map<String, INodeFileUnderConstruction> nodes =
+ leaseManager.getINodesUnderConstruction();
+ out.writeInt(nodes.size()); // write the size
+ for (Map.Entry<String, INodeFileUnderConstruction> entry
+ : nodes.entrySet()) {
+ FSImageSerialization.writeINodeUnderConstruction(
+ out, entry.getValue(), entry.getKey());
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Mon Dec 3 18:04:51 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.util.StringUtils;
@@ -46,8 +47,33 @@ import com.google.common.primitives.Sign
*/
@InterfaceAudience.Private
public abstract class INode implements Comparable<byte[]> {
+ /** A dummy INode which can be used as a probe object. */
+ public static final INode DUMMY = new INode() {
+ @Override
+ int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info) {
+ throw new UnsupportedOperationException();
+ }
+ @Override
+ long[] computeContentSummary(long[] summary) {
+ throw new UnsupportedOperationException();
+ }
+ @Override
+ DirCounts spaceConsumedInTree(DirCounts counts) {
+ throw new UnsupportedOperationException();
+ }
+ };
static final ReadOnlyList<INode> EMPTY_READ_ONLY_LIST
= ReadOnlyList.Util.emptyList();
+ /**
+ * Assert that the snapshot parameter must be null since
+ * this class only take care current state.
+ * Subclasses should override the methods for handling the snapshot states.
+ */
+ static void assertNull(Snapshot snapshot) {
+ if (snapshot != null) {
+ throw new AssertionError("snapshot is not null: " + snapshot);
+ }
+ }
/** Wrapper of two counters for namespace consumed and diskspace consumed. */
@@ -120,9 +146,12 @@ public abstract class INode implements C
* should not modify it.
*/
private long permission = 0L;
- protected INodeDirectory parent = null;
- protected long modificationTime = 0L;
- protected long accessTime = 0L;
+ INodeDirectory parent = null;
+ private long modificationTime = 0L;
+ private long accessTime = 0L;
+
+ /** For creating the a {@link #DUMMY} object. */
+ private INode() {}
private INode(byte[] name, long permission, INodeDirectory parent,
long modificationTime, long accessTime) {
@@ -149,8 +178,8 @@ public abstract class INode implements C
/** @param other Other node to be copied */
INode(INode other) {
- this(other.getLocalNameBytes(), other.permission, other.getParent(),
- other.getModificationTime(), other.getAccessTime());
+ this(other.name, other.permission, other.parent,
+ other.modificationTime, other.accessTime);
}
/**
@@ -290,13 +319,13 @@ public abstract class INode implements C
* Set local file name
*/
public void setLocalName(String name) {
- this.name = DFSUtil.string2Bytes(name);
+ setLocalName(DFSUtil.string2Bytes(name));
}
/**
* Set local file name
*/
- void setLocalName(byte[] name) {
+ public void setLocalName(byte[] name) {
this.name = name;
}
@@ -316,7 +345,7 @@ public abstract class INode implements C
* Get parent directory
* @return parent INode
*/
- INodeDirectory getParent() {
+ public INodeDirectory getParent() {
return this.parent;
}
@@ -336,17 +365,21 @@ public abstract class INode implements C
/**
* Set last modification time of inode.
*/
- public void setModificationTime(long modtime) {
+ public void updateModificationTime(long modtime) {
assert isDirectory();
if (this.modificationTime <= modtime) {
- this.modificationTime = modtime;
+ setModificationTime(modtime);
}
}
+ void cloneModificationTime(INode that) {
+ this.modificationTime = that.modificationTime;
+ }
+
/**
* Always set the last modification time of inode.
*/
- void setModificationTimeForce(long modtime) {
+ void setModificationTime(long modtime) {
this.modificationTime = modtime;
}
@@ -431,11 +464,11 @@ public abstract class INode implements C
return buf.toString();
}
- public boolean removeNode() {
+ public boolean removeNode(Snapshot latestSnapshot) {
if (parent == null) {
return false;
} else {
- parent.removeChild(this);
+ parent.removeChild(this, latestSnapshot);
parent = null;
return true;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Mon Dec 3 18:04:51 2012
@@ -54,7 +54,7 @@ public class INodeDirectory extends INod
}
return (INodeDirectory)inode;
}
-
+
protected static final int DEFAULT_FILES_PER_DIRECTORY = 5;
final static String ROOT_NAME = "";
@@ -99,32 +99,56 @@ public class INodeDirectory extends INod
}
}
- private int searchChildren(INode inode) {
+ public int searchChildren(INode inode) {
return Collections.binarySearch(children, inode.getLocalNameBytes());
}
- INode removeChild(INode node) {
+ public int searchChildrenForExistingINode(INode inode) {
+ final int i = searchChildren(inode);
+ if (i < 0) {
+ throw new AssertionError("Child not found: inode=" + inode);
+ }
+ return i;
+ }
+
+ public INode removeChild(INode node, Snapshot latestSnapshot) {
assertChildrenNonNull();
+
+ if (latestSnapshot != null) {
+ final INodeDirectoryWithSnapshot dir
+ = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
+ return dir.removeChild(node, latestSnapshot);
+ }
+
final int i = searchChildren(node);
return i >= 0? children.remove(i): null;
}
-
/** Replace a child that has the same name as newChild by newChild.
*
* @param newChild Child node to be added
*/
- void replaceChild(INode newChild) {
+ public INode replaceChild(INodeDirectory newChild, Snapshot latestSnapshot) {
assertChildrenNonNull();
- final int low = searchChildren(newChild);
- if (low>=0) { // an old child exists so replace by the newChild
- children.set(low, newChild);
- } else {
- throw new IllegalArgumentException("No child exists to be replaced");
+ if (latestSnapshot != null) {
+ final INodeDirectoryWithSnapshot dir
+ = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
+ return dir.replaceChild(newChild, latestSnapshot);
+ }
+
+ // find the old child and replace it
+ final int low = searchChildrenForExistingINode(newChild);
+ final INode oldChild = children.set(low, newChild);
+ // set the parent of the children of the child.
+ for(INode i : newChild.getChildrenList(null)) {
+ i.parent = newChild;
}
+ return oldChild;
}
- private INode getChild(byte[] name, Snapshot snapshot) {
+ public INode getChild(byte[] name, Snapshot snapshot) {
+ assertNull(snapshot);
+
final ReadOnlyList<INode> c = getChildrenList(snapshot);
final int i = ReadOnlyList.Util.binarySearch(c, name);
return i < 0? null: c.get(i);
@@ -361,7 +385,14 @@ public class INodeDirectory extends INod
* @return false if the child with this name already exists;
* otherwise, return true;
*/
- public boolean addChild(final INode node, final boolean setModTime) {
+ public boolean addChild(final INode node, final boolean setModTime,
+ Snapshot latestSnapshot) {
+ if (latestSnapshot != null) {
+ final INodeDirectoryWithSnapshot dir
+ = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
+ return dir.addChild(node, setModTime, latestSnapshot);
+ }
+
if (children == null) {
children = new ArrayList<INode>(DEFAULT_FILES_PER_DIRECTORY);
}
@@ -372,8 +403,9 @@ public class INodeDirectory extends INod
node.parent = this;
children.add(-low - 1, node);
// update modification time of the parent directory
- if (setModTime)
- setModificationTime(node.getModificationTime());
+ if (setModTime) {
+ updateModificationTime(node.getModificationTime());
+ }
if (node.getGroupName() == null) {
node.setGroup(getGroupName());
}
@@ -400,20 +432,28 @@ public class INodeDirectory extends INod
}
newNode.setLocalName(pathComponents[pathComponents.length - 1]);
// insert into the parent children list
- INodeDirectory parent = getParent(pathComponents);
- return parent.addChild(newNode, true);
+ INodesInPath inodes = getExistingPathINodes(pathComponents, 2, false);
+ INodeDirectory parent = INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
+ return parent.addChild(newNode, true, inodes.getLatestSnapshot());
}
INodeDirectory getParent(byte[][] pathComponents
) throws FileNotFoundException, PathIsNotDirectoryException,
UnresolvedLinkException {
+ return (INodeDirectory)getParentINodesInPath(pathComponents).getINode(0);
+ }
+
+ INodesInPath getParentINodesInPath(byte[][] pathComponents
+ ) throws FileNotFoundException, PathIsNotDirectoryException,
+ UnresolvedLinkException {
if (pathComponents.length < 2) // add root
return null;
// Gets the parent INode
INodesInPath inodes = getExistingPathINodes(pathComponents, 2, false);
- return INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
+ INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
+ return inodes;
}
-
+
@Override
DirCounts spaceConsumedInTree(DirCounts counts) {
counts.nsCount += 1;
@@ -462,10 +502,11 @@ public class INodeDirectory extends INod
* Note that the returned list is never null.
*/
public ReadOnlyList<INode> getChildrenList(final Snapshot snapshot) {
- //TODO: use snapshot to select children list
+ assertNull(snapshot);
return children == null ? EMPTY_READ_ONLY_LIST
: ReadOnlyList.Util.asReadOnlyList(children);
}
+
/** Set the children list. */
public void setChildren(List<INode> children) {
this.children = children;
@@ -490,7 +531,7 @@ public class INodeDirectory extends INod
* {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)}.
* Contains INodes information resolved from a given path.
*/
- static class INodesInPath {
+ public static class INodesInPath {
/**
* Array with the specified number of INodes resolved for a given path.
*/
@@ -570,7 +611,7 @@ public class INodeDirectory extends INod
}
/** @return the i-th inode. */
- INode getINode(int i) {
+ public INode getINode(int i) {
return inodes[i];
}
@@ -676,4 +717,13 @@ public class INodeDirectory extends INod
}
}
}
+
+ /**
+ * Get last modification time of inode.
+ * @return access time
+ */
+ public long getModificationTime(Snapshot snapshot) {
+ assertNull(snapshot);
+ return getModificationTime();
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Mon Dec 3 18:04:51 2012
@@ -89,16 +89,22 @@ public class INodeFile extends INode imp
INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
short replication, long modificationTime,
long atime, long preferredBlockSize) {
- super(permissions, modificationTime, atime);
+ this(null, permissions, modificationTime, atime, blklist, replication,
+ preferredBlockSize);
+ }
+
+ INodeFile(byte[] name, PermissionStatus permissions, long mtime, long atime,
+ BlockInfo[] blklist, short replication, long preferredBlockSize) {
+ super(name, permissions, null, mtime, atime);
header = HeaderFormat.combineReplication(header, replication);
header = HeaderFormat.combinePreferredBlockSize(header, preferredBlockSize);
this.blocks = blklist;
}
- protected INodeFile(INodeFile f) {
- this(f.getPermissionStatus(), f.getBlocks(), f.getFileReplication(),
- f.getModificationTime(), f.getAccessTime(), f.getPreferredBlockSize());
- this.setLocalName(f.getLocalNameBytes());
+ protected INodeFile(INodeFile that) {
+ super(that);
+ this.header = that.header;
+ this.blocks = that.blocks;
}
/** @return true unconditionally. */
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java Mon Dec 3 18:04:51 2012
@@ -72,9 +72,8 @@ public class INodeFileUnderConstruction
String clientName,
String clientMachine,
DatanodeDescriptor clientNode) {
- super(perm, blocks, blockReplication, modificationTime, modificationTime,
- preferredBlockSize);
- setLocalName(name);
+ super(name, perm, modificationTime, modificationTime,
+ blocks, blockReplication, preferredBlockSize);
this.clientName = clientName;
this.clientMachine = clientMachine;
this.clientNode = clientNode;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Mon Dec 3 18:04:51 2012
@@ -331,22 +331,19 @@ public class LeaseManager {
}
}
- synchronized void changeLease(String src, String dst,
- String overwrite, String replaceBy) {
+ synchronized void changeLease(String src, String dst) {
if (LOG.isDebugEnabled()) {
LOG.debug(getClass().getSimpleName() + ".changelease: " +
- " src=" + src + ", dest=" + dst +
- ", overwrite=" + overwrite +
- ", replaceBy=" + replaceBy);
+ " src=" + src + ", dest=" + dst);
}
- final int len = overwrite.length();
+ final int len = src.length();
for(Map.Entry<String, Lease> entry
: findLeaseWithPrefixPath(src, sortedLeasesByPath).entrySet()) {
final String oldpath = entry.getKey();
final Lease lease = entry.getValue();
- //overwrite must be a prefix of oldpath
- final String newpath = replaceBy + oldpath.substring(len);
+ // replace stem of src with new destination
+ final String newpath = dst + oldpath.substring(len);
if (LOG.isDebugEnabled()) {
LOG.debug("changeLease: replacing " + oldpath + " with " + newpath);
}
@@ -429,6 +426,26 @@ public class LeaseManager {
}
}
+ /**
+ * Get the list of inodes corresponding to valid leases.
+ * @return list of inodes
+ * @throws UnresolvedLinkException
+ */
+ Map<String, INodeFileUnderConstruction> getINodesUnderConstruction() {
+ Map<String, INodeFileUnderConstruction> inodes =
+ new TreeMap<String, INodeFileUnderConstruction>();
+ for (String p : sortedLeasesByPath.keySet()) {
+ // verify that path exists in namespace
+ try {
+ INode node = fsnamesystem.dir.getINode(p);
+ inodes.put(p, INodeFileUnderConstruction.valueOf(node, p));
+ } catch (IOException ioe) {
+ LOG.error(ioe);
+ }
+ }
+ return inodes;
+ }
+
/** Check the leases beginning from the oldest.
* @return true is sync is needed.
*/
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Mon Dec 3 18:04:51 2012
@@ -598,11 +598,7 @@ public class NameNode {
String nsId = getNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
this.haEnabled = HAUtil.isHAEnabled(conf, nsId);
- if (!haEnabled) {
- state = ACTIVE_STATE;
- } else {
- state = STANDBY_STATE;
- }
+ state = createHAState();
this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf);
this.haContext = createHAContext();
try {
@@ -619,6 +615,10 @@ public class NameNode {
}
}
+ protected HAState createHAState() {
+ return !haEnabled ? ACTIVE_STATE : STANDBY_STATE;
+ }
+
protected HAContext createHAContext() {
return new NameNodeHAContext();
}
@@ -1298,7 +1298,7 @@ public class NameNode {
* before exit.
* @throws ExitException thrown only for testing.
*/
- private synchronized void doImmediateShutdown(Throwable t)
+ protected synchronized void doImmediateShutdown(Throwable t)
throws ExitException {
String message = "Error encountered requiring NN shutdown. " +
"Shutting down immediately.";
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Mon Dec 3 18:04:51 2012
@@ -102,7 +102,7 @@ class NamenodeJspHelper {
long usedNonHeap = (totalNonHeap * 100) / commitedNonHeap;
String str = "<div>" + inodes + " files and directories, " + blocks + " blocks = "
- + (inodes + blocks) + " total";
+ + (inodes + blocks) + " total filesystem objects";
if (maxobjects != 0) {
long pct = ((inodes + blocks) * 100) / maxobjects;
str += " / " + maxobjects + " (" + pct + "%)";
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Mon Dec 3 18:04:51 2012
@@ -65,6 +65,19 @@ public class INodeDirectorySnapshottable
}
return (INodeDirectorySnapshottable)dir;
}
+
+ public static Snapshot findLatestSnapshot(INode inode) {
+ Snapshot latest = null;
+ for(; inode != null; inode = inode.getParent()) {
+ if (inode instanceof INodeDirectorySnapshottable) {
+ final Snapshot s = ((INodeDirectorySnapshottable)inode).getLastSnapshot();
+ if (Snapshot.ID_COMPARATOR.compare(latest, s) < 0) {
+ latest = s;
+ }
+ }
+ }
+ return latest;
+ }
/** Snapshots of this directory in ascending order of snapshot id. */
private final List<Snapshot> snapshots = new ArrayList<Snapshot>();
@@ -196,8 +209,8 @@ public class INodeDirectorySnapshottable
//set modification time
final long timestamp = Time.now();
- s.getRoot().setModificationTime(timestamp);
- setModificationTime(timestamp);
+ s.getRoot().updateModificationTime(timestamp);
+ updateModificationTime(timestamp);
return s;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java Mon Dec 3 18:04:51 2012
@@ -19,10 +19,15 @@ package org.apache.hadoop.hdfs.server.na
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+
+import com.google.common.base.Preconditions;
/** The directory with snapshots. */
public class INodeDirectoryWithSnapshot extends INodeDirectory {
@@ -250,9 +255,223 @@ public class INodeDirectoryWithSnapshot
+ "\n deleted=" + toString(deleted);
}
}
+
+ private class SnapshotDiff implements Comparable<Snapshot> {
+ /** The snapshot will obtain after applied this diff. */
+ final Snapshot snapshot;
+ /** The size of the children list which is never changed. */
+ final int size;
+ /**
+ * Posterior diff is the diff happened after this diff.
+ * The posterior diff should be first applied to obtain the posterior
+ * snapshot and then apply this diff in order to obtain this snapshot.
+ * If the posterior diff is null, the posterior state is the current state.
+ */
+ private SnapshotDiff posteriorDiff;
+ /** The data of this diff. */
+ private final Diff diff = new Diff();
+ /** The snapshot version of the inode. */
+ private INode snapshotINode;
+
+ SnapshotDiff(Snapshot snapshot, int size) {
+ if (size < 0) {
+ throw new HadoopIllegalArgumentException("size = " + size + " < 0");
+ }
+ this.snapshot = snapshot;
+ this.size = size;
+ }
+
+ @Override
+ public int compareTo(final Snapshot that_snapshot) {
+ return Snapshot.ID_COMPARATOR.compare(this.snapshot, that_snapshot);
+ }
+
+ /**
+ * @return The children list of a directory in a snapshot.
+ * Since the snapshot is read-only, the logical view of the list is
+ * never changed although the internal data structure may mutate.
+ */
+ ReadOnlyList<INode> getChildrenList() {
+ return new ReadOnlyList<INode>() {
+ private List<INode> children = null;
+
+ private List<INode> initChildren() {
+ if (children == null) {
+ final ReadOnlyList<INode> posterior = posteriorDiff != null?
+ posteriorDiff.getChildrenList()
+ : INodeDirectoryWithSnapshot.this.getChildrenList(null);
+ children = diff.apply2Current(ReadOnlyList.Util.asList(posterior));
+ }
+ return children;
+ }
+
+ @Override
+ public Iterator<INode> iterator() {
+ return initChildren().iterator();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return size == 0;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public INode get(int i) {
+ return initChildren().get(i);
+ }
+ };
+ }
+
+ INode getChild(byte[] name) {
+ final INode i = diff.accessPrevious(name, INode.DUMMY);
+ if (i != INode.DUMMY) {
+ // this diff is able to find it
+ return i;
+ } else {
+ // should return the posterior INode.
+ return posteriorDiff != null? posteriorDiff.getChild(name)
+ : INodeDirectoryWithSnapshot.this.getChild(name, null);
+ }
+ }
+ }
+
+ /** Replace the given directory to an {@link INodeDirectoryWithSnapshot}. */
+ public static INodeDirectoryWithSnapshot replaceDir(INodeDirectory oldDir,
+ Snapshot latestSnapshot) {
+ Preconditions.checkArgument(!(oldDir instanceof INodeDirectoryWithSnapshot),
+ "oldDir is already an INodeDirectoryWithSnapshot, oldDir=%s", oldDir);
+
+ final INodeDirectory parent = oldDir.getParent();
+ Preconditions.checkArgument(parent != null,
+ "parent is null, oldDir=%s", oldDir);
+
+ final INodeDirectoryWithSnapshot newDir = new INodeDirectoryWithSnapshot(
+ oldDir, latestSnapshot);
+ parent.replaceChild(newDir, null);
+ return newDir;
+ }
+
+ /** Diff list sorted by snapshot IDs, i.e. in chronological order. */
+ private final List<SnapshotDiff> diffs = new ArrayList<SnapshotDiff>();
- INodeDirectoryWithSnapshot(String name, INodeDirectory dir) {
- super(name, dir.getPermissionStatus());
- parent = dir;
+ INodeDirectoryWithSnapshot(INodeDirectory that, Snapshot s) {
+ super(that);
+
+ // add a diff for the snapshot
+ addSnapshotDiff(s, that.getChildrenList(null).size());
+ }
+
+ INodeDirectoryWithSnapshot(String name, INodeDirectory dir, Snapshot s) {
+ this(dir, s);
+ setLocalName(name);
+ setParent(dir);
+ }
+
+ SnapshotDiff addSnapshotDiff(Snapshot snapshot, int childrenSize) {
+ final SnapshotDiff d = new SnapshotDiff(snapshot, childrenSize);
+ diffs.add(d);
+ return d;
+ }
+
+ /**
+ * Check if the latest snapshot diff exist. If not, add it.
+ * @return the latest snapshot diff, which is never null.
+ */
+ private SnapshotDiff checkAndAddLatestSnapshotDiff(Snapshot latest) {
+ final SnapshotDiff last = getLastSnapshotDiff();
+ if (last != null && last.snapshot.equals(latest)) {
+ return last;
+ }
+
+ final int size = getChildrenList(null).size();
+ final SnapshotDiff d = addSnapshotDiff(latest, size);
+ if (last != null) {
+ last.posteriorDiff = d;
+ }
+ return d;
+ }
+
+ Diff getLatestDiff(Snapshot latest) {
+ return checkAndAddLatestSnapshotDiff(latest).diff;
+ }
+
+ /**
+ * @return the diff corresponding to the snapshot.
+ * When the diff is not found, it means that the current state and
+ * the snapshot state are the same.
+ */
+ SnapshotDiff getSnapshotDiff(Snapshot snapshot) {
+ if (snapshot == null) {
+ return null;
+ }
+ final int i = Collections.binarySearch(diffs, snapshot);
+ if (i >= 0) {
+ // exact match
+ return diffs.get(i);
+ } else {
+ // Exact match not found means that there were no changes between
+ // given snapshot and the next state so that the diff for the given
+ // snapshot is not recorded. Thus, use the next state.
+ final int j = -i - 1;
+ return j < diffs.size()? diffs.get(j): null;
+ }
+ }
+
+ SnapshotDiff getLastSnapshotDiff() {
+ return diffs.get(diffs.size() - 1);
+ }
+
+ @Override
+ public ReadOnlyList<INode> getChildrenList(Snapshot snapshot) {
+ final SnapshotDiff diff = getSnapshotDiff(snapshot);
+ if (diff != null) {
+ return diff.getChildrenList();
+ }
+ return super.getChildrenList(null);
+ }
+
+ @Override
+ public INode getChild(byte[] name, Snapshot snapshot) {
+ final SnapshotDiff diff = getSnapshotDiff(snapshot);
+ if (diff != null) {
+ return diff.getChild(name);
+ }
+ return super.getChild(name, null);
+ }
+
+ @Override
+ public boolean addChild(INode inode, boolean setModTime,
+ Snapshot latestSnapshot) {
+ getLatestDiff(latestSnapshot).create(inode);
+ return super.addChild(inode, setModTime, null);
+ }
+
+ @Override
+ public INode removeChild(INode inode, Snapshot latestSnapshot) {
+ getLatestDiff(latestSnapshot).delete(inode);
+ return super.removeChild(inode, null);
+ }
+
+ @Override
+ public INode replaceChild(INodeDirectory newChild, Snapshot latestSnapshot) {
+ final INode oldChild = super.replaceChild(newChild, null);
+ final Diff diff = getLatestDiff(latestSnapshot);
+ diff.delete(oldChild);
+ diff.create(newChild);
+ return oldChild;
+ }
+
+ @Override
+ public long getModificationTime(Snapshot snapshot) {
+ final SnapshotDiff diff = getSnapshotDiff(snapshot);
+ if (diff != null) {
+ return diff.snapshotINode.getModificationTime();
+ }
+ return getModificationTime();
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java Mon Dec 3 18:04:51 2012
@@ -109,7 +109,7 @@ public class INodeFileWithLink extends I
this.setFileReplication(maxReplication);
this.next = null;
// clear parent
- parent = null;
+ setParent(null);
}
return 1;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java Mon Dec 3 18:04:51 2012
@@ -44,7 +44,7 @@ public class Snapshot implements Compara
Snapshot(int id, String name, INodeDirectorySnapshottable dir) {
this.id = id;
- this.root = new INodeDirectoryWithSnapshot(name, dir);
+ this.root = new INodeDirectoryWithSnapshot(name, dir, this);
}
/** @return the root directory of the snapshot. */
@@ -58,6 +58,21 @@ public class Snapshot implements Compara
}
@Override
+ public boolean equals(Object that) {
+ if (this == that) {
+ return true;
+ } else if (that == null || !(that instanceof Snapshot)) {
+ return false;
+ }
+ return this.id == ((Snapshot)that).id;
+ }
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + ":" + root.getLocalName();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Mon Dec 3 18:04:51 2012
@@ -24,12 +24,8 @@ import java.util.concurrent.atomic.Atomi
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
-import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction;
-import org.apache.hadoop.hdfs.server.namenode.INodeSymlink;
-import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
/**
* Manage snapshottable directories and their snapshots.
@@ -68,7 +64,9 @@ public class SnapshotManager implements
*/
public void setSnapshottable(final String path, final int snapshotQuota
) throws IOException {
- final INodeDirectory d = INodeDirectory.valueOf(fsdir.getINode(path), path);
+ final INodesInPath inodesInPath = fsdir.getINodesInPath(path);
+ final INodeDirectory d = INodeDirectory.valueOf(
+ inodesInPath.getINode(0), path);
if (d.isSnapshottable()) {
//The directory is already a snapshottable directory.
((INodeDirectorySnapshottable)d).setSnapshotQuota(snapshotQuota);
@@ -77,7 +75,7 @@ public class SnapshotManager implements
final INodeDirectorySnapshottable s
= INodeDirectorySnapshottable.newInstance(d, snapshotQuota);
- fsdir.replaceINodeDirectory(path, d, s);
+ fsdir.replaceINodeDirectory(path, d, s, inodesInPath.getLatestSnapshot());
snapshottables.add(s);
numSnapshottableDirs.getAndIncrement();
@@ -90,15 +88,16 @@ public class SnapshotManager implements
*/
public void resetSnapshottable(final String path
) throws IOException {
+ final INodesInPath inodesInPath = fsdir.getINodesInPath(path);
final INodeDirectorySnapshottable s = INodeDirectorySnapshottable.valueOf(
- fsdir.getINode(path), path);
+ inodesInPath.getINode(0), path);
if (s.getNumSnapshots() > 0) {
throw new SnapshotException("The directory " + path + " has snapshot(s). "
+ "Please redo the operation after removing all the snapshots.");
}
final INodeDirectory d = new INodeDirectory(s);
- fsdir.replaceINodeDirectory(path, s, d);
+ fsdir.replaceINodeDirectory(path, s, d, inodesInPath.getLatestSnapshot());
snapshottables.remove(s);
numSnapshottableDirs.getAndDecrement();
@@ -121,9 +120,8 @@ public class SnapshotManager implements
// Find the source root directory path where the snapshot is taken.
final INodeDirectorySnapshottable srcRoot
= INodeDirectorySnapshottable.valueOf(fsdir.getINode(path), path);
- final Snapshot s = srcRoot.addSnapshot(snapshotID, snapshotName);
- new SnapshotCreation().processRecursively(srcRoot, s.getRoot());
-
+ srcRoot.addSnapshot(snapshotID, snapshotName);
+
//create success, update id
snapshotID++;
numSnapshots.getAndIncrement();
@@ -154,85 +152,6 @@ public class SnapshotManager implements
srcRoot.renameSnapshot(path, oldSnapshotName, newSnapshotName);
}
- /**
- * Create a snapshot of subtrees by recursively coping the directory
- * structure from the source directory to the snapshot destination directory.
- * This creation algorithm requires O(N) running time and O(N) memory,
- * where N = # files + # directories + # symlinks.
- */
- class SnapshotCreation {
- /** Process snapshot creation recursively. */
- private void processRecursively(final INodeDirectory srcDir,
- final INodeDirectory dstDir) throws IOException {
- final ReadOnlyList<INode> children = srcDir.getChildrenList(null);
- if (!children.isEmpty()) {
- final List<INode> inodes = new ArrayList<INode>(children.size());
- for(final INode c : new ArrayList<INode>(ReadOnlyList.Util.asList(children))) {
- final INode i;
- if (c == null) {
- i = null;
- } else if (c instanceof INodeDirectory) {
- //also handle INodeDirectoryWithQuota
- i = processINodeDirectory((INodeDirectory)c);
- } else if (c instanceof INodeFileUnderConstruction) {
- //TODO: support INodeFileUnderConstruction
- throw new IOException("Not yet supported.");
- } else if (c instanceof INodeFile) {
- i = processINodeFile(srcDir, (INodeFile)c);
- } else if (c instanceof INodeSymlink) {
- i = new INodeSymlink((INodeSymlink)c);
- } else {
- throw new AssertionError("Unknow INode type: " + c.getClass()
- + ", inode = " + c);
- }
- i.setParent(dstDir);
- inodes.add(i);
- }
- dstDir.setChildren(inodes);
- }
- }
-
- /**
- * Create destination INodeDirectory and make the recursive call.
- * @return destination INodeDirectory.
- */
- private INodeDirectory processINodeDirectory(final INodeDirectory srcChild
- ) throws IOException {
- final INodeDirectory dstChild = new INodeDirectory(srcChild);
- dstChild.setChildren(null);
- processRecursively(srcChild, dstChild);
- return dstChild;
- }
-
- /**
- * Create destination INodeFileSnapshot and update source INode type.
- * @return destination INodeFileSnapshot.
- */
- private INodeFileSnapshot processINodeFile(final INodeDirectory parent,
- final INodeFile file) {
- final INodeFileSnapshot snapshot = new INodeFileSnapshot(
- file, file.computeFileSize(true));
-
- final INodeFileWithLink srcWithLink;
- //check source INode type
- if (file instanceof INodeFileWithLink) {
- srcWithLink = (INodeFileWithLink)file;
- } else {
- //source is an INodeFile, replace the source.
- srcWithLink = new INodeFileWithLink(file);
- file.removeNode();
- parent.addChild(srcWithLink, false);
-
- //update block map
- namesystem.getBlockManager().addBlockCollection(srcWithLink);
- }
-
- //insert the snapshot to src's linked list.
- srcWithLink.insert(snapshot);
- return snapshot;
- }
- }
-
@Override
public long getNumSnapshottableDirs() {
return numSnapshottableDirs.get();
@@ -243,4 +162,4 @@ public class SnapshotManager implements
return numSnapshots.get();
}
-}
\ No newline at end of file
+}
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1415804-1416602