You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2012/12/03 23:43:15 UTC
svn commit: r1416712 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/
src/main/native/ s...
Author: suresh
Date: Mon Dec 3 22:43:00 2012
New Revision: 1416712
URL: http://svn.apache.org/viewvc?rev=1416712&view=rev
Log:
Reverting the previous merge r1416603 which committed some extra changes
Removed:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1415804-1416602
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Dec 3 22:43:00 2012
@@ -176,9 +176,6 @@ Trunk (Unreleased)
HDFS-4209. Clean up the addNode/addChild/addChildNoQuotaCheck methods in
FSDirectory and INodeDirectory. (szetszwo)
- HDFS-3358. Specify explicitly that the NN UI status total is talking
- of persistent objects on heap. (harsh)
-
OPTIMIZATIONS
BUG FIXES
@@ -649,8 +646,6 @@ Release 2.0.3-alpha - Unreleased
of it is undefined after the iteration or modifications of the map.
(szetszwo)
- HDFS-4231. BackupNode: Introduce BackupState. (shv)
-
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -2040,11 +2035,6 @@ Release 0.23.6 - UNRELEASED
BUG FIXES
- HDFS-4247. saveNamespace should be tolerant of dangling lease (daryn)
-
- HDFS-4248. Renaming directories may incorrectly remove the paths in leases
- under the tree. (daryn via szetszwo)
-
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1415804-1416602
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Mon Dec 3 22:43:00 2012
@@ -24,7 +24,6 @@ import java.net.SocketTimeoutException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.NameNodeProxies;
@@ -36,7 +35,6 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
import org.apache.hadoop.hdfs.server.protocol.FenceResponse;
import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
@@ -416,23 +414,14 @@ public class BackupNode extends NameNode
+ HdfsConstants.LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion();
return nsInfo;
}
-
+
@Override
- protected String getNameServiceId(Configuration conf) {
- return DFSUtil.getBackupNameServiceId(conf);
- }
-
- protected HAState createHAState() {
- return new BackupState();
- }
-
- @Override // NameNode
protected NameNodeHAContext createHAContext() {
return new BNHAContext();
}
-
+
private class BNHAContext extends NameNodeHAContext {
- @Override // NameNodeHAContext
+ @Override // NameNode
public void checkOperation(OperationCategory op)
throws StandbyException {
if (op == OperationCategory.UNCHECKED ||
@@ -446,42 +435,10 @@ public class BackupNode extends NameNode
throw new StandbyException(msg);
}
}
-
- @Override // NameNodeHAContext
- public void prepareToStopStandbyServices() throws ServiceFailedException {
- }
-
- /**
- * Start services for BackupNode.
- * <p>
- * The following services should be muted
- * (not run or not pass any control commands to DataNodes)
- * on BackupNode:
- * {@link LeaseManager.Monitor} protected by SafeMode.
- * {@link BlockManager.ReplicationMonitor} protected by SafeMode.
- * {@link HeartbeatManager.Monitor} protected by SafeMode.
- * {@link DecommissionManager.Monitor} need to prohibit refreshNodes().
- * {@link PendingReplicationBlocks.PendingReplicationMonitor} harmless,
- * because ReplicationMonitor is muted.
- */
- @Override
- public void startActiveServices() throws IOException {
- try {
- namesystem.startActiveServices();
- } catch (Throwable t) {
- doImmediateShutdown(t);
- }
- }
-
- @Override
- public void stopActiveServices() throws IOException {
- try {
- if (namesystem != null) {
- namesystem.stopActiveServices();
- }
- } catch (Throwable t) {
- doImmediateShutdown(t);
- }
- }
+ }
+
+ @Override
+ protected String getNameServiceId(Configuration conf) {
+ return DFSUtil.getBackupNameServiceId(conf);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Mon Dec 3 22:43:00 2012
@@ -383,7 +383,7 @@ public class FSDirectory implements Clos
writeLock();
try {
// file is closed
- file.setModificationTime(now);
+ file.setModificationTimeForce(now);
fsImage.getEditLog().logCloseFile(path, file);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.closeFile: "
@@ -585,10 +585,8 @@ public class FSDirectory implements Clos
+ src + " is renamed to " + dst);
}
// update modification time of dst and the parent of src
- srcInodes[srcInodes.length-2].updateModificationTime(timestamp);
- dstInodes[dstInodes.length-2].updateModificationTime(timestamp);
- // update moved leases with new filename
- getFSNamesystem().unprotectedChangeLease(src, dst);
+ srcInodes[srcInodes.length-2].setModificationTime(timestamp);
+ dstInodes[dstInodes.length-2].setModificationTime(timestamp);
return true;
}
} finally {
@@ -752,10 +750,8 @@ public class FSDirectory implements Clos
"DIR* FSDirectory.unprotectedRenameTo: " + src
+ " is renamed to " + dst);
}
- srcInodes[srcInodes.length - 2].updateModificationTime(timestamp);
- dstInodes[dstInodes.length - 2].updateModificationTime(timestamp);
- // update moved lease with new filename
- getFSNamesystem().unprotectedChangeLease(src, dst);
+ srcInodes[srcInodes.length - 2].setModificationTime(timestamp);
+ dstInodes[dstInodes.length - 2].setModificationTime(timestamp);
// Collect the blocks and remove the lease for previous dst
int filesDeleted = 0;
@@ -990,12 +986,12 @@ public class FSDirectory implements Clos
if(nodeToRemove == null) continue;
nodeToRemove.setBlocks(null);
- trgParent.removeChild(nodeToRemove, trgINodesInPath.getLatestSnapshot());
+ trgParent.removeChild(nodeToRemove);
count++;
}
- trgInode.setModificationTime(timestamp);
- trgParent.updateModificationTime(timestamp);
+ trgInode.setModificationTimeForce(timestamp);
+ trgParent.setModificationTime(timestamp);
// update quota on the parent directory ('count' files removed, 0 space)
unprotectedUpdateCount(trgINodesInPath, trgINodes.length-1, -count, 0);
}
@@ -1133,7 +1129,7 @@ public class FSDirectory implements Clos
return 0;
}
// set the parent's modification time
- inodes[inodes.length - 2].updateModificationTime(mtime);
+ inodes[inodes.length - 2].setModificationTime(mtime);
int filesRemoved = targetNode.collectSubtreeBlocksAndClear(collectedBlocks);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
@@ -1169,10 +1165,10 @@ public class FSDirectory implements Clos
/**
* Replaces the specified INode.
*/
- private void replaceINodeUnsynced(String path, INode oldnode, INode newnode,
- Snapshot latestSnapshot) throws IOException {
+ private void replaceINodeUnsynced(String path, INode oldnode, INode newnode
+ ) throws IOException {
//remove the old node from the namespace
- if (!oldnode.removeNode(latestSnapshot)) {
+ if (!oldnode.removeNode()) {
final String mess = "FSDirectory.replaceINodeUnsynced: failed to remove "
+ path;
NameNode.stateChangeLog.warn("DIR* " + mess);
@@ -1187,10 +1183,10 @@ public class FSDirectory implements Clos
* Replaces the specified INodeDirectory.
*/
public void replaceINodeDirectory(String path, INodeDirectory oldnode,
- INodeDirectory newnode, Snapshot latestSnapshot) throws IOException {
+ INodeDirectory newnode) throws IOException {
writeLock();
try {
- replaceINodeUnsynced(path, oldnode, newnode, latestSnapshot);
+ replaceINodeUnsynced(path, oldnode, newnode);
//update children's parent directory
for(INode i : newnode.getChildrenList(null)) {
@@ -1204,11 +1200,11 @@ public class FSDirectory implements Clos
/**
* Replaces the specified INodeFile with the specified one.
*/
- public void replaceNode(String path, INodeFile oldnode, INodeFile newnode,
- Snapshot latestSnapshot) throws IOException {
+ public void replaceNode(String path, INodeFile oldnode, INodeFile newnode
+ ) throws IOException {
writeLock();
try {
- replaceINodeUnsynced(path, oldnode, newnode, latestSnapshot);
+ replaceINodeUnsynced(path, oldnode, newnode);
//Currently, oldnode and newnode are assumed to contain the same blocks.
//Otherwise, blocks need to be removed from the blocksMap.
@@ -1277,9 +1273,13 @@ public class FSDirectory implements Clos
String srcs = normalizePath(src);
readLock();
try {
- final INodesInPath inodesInPath = rootDir.getINodesInPath(srcs, resolveLink);
- final INode i = inodesInPath.getINode(0);
- return i == null? null: createFileStatus(HdfsFileStatus.EMPTY_NAME, i);
+ INode targetNode = rootDir.getNode(srcs, resolveLink);
+ if (targetNode == null) {
+ return null;
+ }
+ else {
+ return createFileStatus(HdfsFileStatus.EMPTY_NAME, targetNode);
+ }
} finally {
readUnlock();
}
@@ -1303,16 +1303,9 @@ public class FSDirectory implements Clos
* Get {@link INode} associated with the file / directory.
*/
public INode getINode(String src) throws UnresolvedLinkException {
- return getINodesInPath(src).getINode(0);
- }
-
- /**
- * Get {@link INode} associated with the file / directory.
- */
- public INodesInPath getINodesInPath(String src) throws UnresolvedLinkException {
readLock();
try {
- return rootDir.getINodesInPath(src, true);
+ return rootDir.getNode(src, true);
} finally {
readUnlock();
}
@@ -1807,8 +1800,7 @@ public class FSDirectory implements Clos
if (inodes[pos-1] == null) {
throw new NullPointerException("Panic: parent does not exist");
}
- final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true,
- inodesInPath.getLatestSnapshot());
+ final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true);
if (!added) {
updateCount(inodesInPath, pos, -counts.getNsCount(), -counts.getDsCount(), true);
}
@@ -1832,8 +1824,7 @@ public class FSDirectory implements Clos
private INode removeLastINode(final INodesInPath inodesInPath) {
final INode[] inodes = inodesInPath.getINodes();
final int pos = inodes.length - 1;
- INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos],
- inodesInPath.getLatestSnapshot());
+ INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos]);
if (removedNode != null) {
INode.DirCounts counts = new INode.DirCounts();
removedNode.spaceConsumedInTree(counts);
@@ -1974,8 +1965,8 @@ public class FSDirectory implements Clos
}
String srcs = normalizePath(src);
- final INodesInPath inodesInPath = rootDir.getMutableINodesInPath(srcs, true);
- final INode[] inodes = inodesInPath.getINodes();
+ final INode[] inodes = rootDir.getMutableINodesInPath(srcs, true)
+ .getINodes();
INodeDirectory dirNode = INodeDirectory.valueOf(inodes[inodes.length-1], srcs);
if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
throw new IllegalArgumentException("Cannot clear namespace quota on root.");
@@ -1997,7 +1988,7 @@ public class FSDirectory implements Clos
INodeDirectory newNode = new INodeDirectory(dirNode);
INodeDirectory parent = (INodeDirectory)inodes[inodes.length-2];
dirNode = newNode;
- parent.replaceChild(newNode, inodesInPath.getLatestSnapshot());
+ parent.replaceChild(newNode);
}
} else {
// a non-quota directory; so replace it with a directory with quota
@@ -2006,7 +1997,7 @@ public class FSDirectory implements Clos
// non-root directory node; parent != null
INodeDirectory parent = (INodeDirectory)inodes[inodes.length-2];
dirNode = newNode;
- parent.replaceChild(newNode, inodesInPath.getLatestSnapshot());
+ parent.replaceChild(newNode);
}
return (oldNsQuota != nsQuota || oldDsQuota != dsQuota) ? dirNode : null;
}
@@ -2070,7 +2061,7 @@ public class FSDirectory implements Clos
assert hasWriteLock();
boolean status = false;
if (mtime != -1) {
- inode.setModificationTime(mtime);
+ inode.setModificationTimeForce(mtime);
status = true;
}
if (atime != -1) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Mon Dec 3 22:43:00 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
@@ -57,7 +58,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp;
-import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.util.Holder;
@@ -246,8 +246,7 @@ public class FSEditLogLoader {
// 3. OP_ADD to open file for append
// See if the file already exists (persistBlocks call)
- final INodesInPath inodesInPath = fsDir.getINodesInPath(addCloseOp.path);
- INodeFile oldFile = toINodeFile(inodesInPath.getINode(0), addCloseOp.path);
+ INodeFile oldFile = getINodeFile(fsDir, addCloseOp.path);
INodeFile newFile = oldFile;
if (oldFile == null) { // this is OP_ADD on a new file (case 1)
// versions > 0 support per file replication
@@ -273,7 +272,7 @@ public class FSEditLogLoader {
}
fsNamesys.prepareFileForWrite(addCloseOp.path, oldFile,
addCloseOp.clientName, addCloseOp.clientMachine, null,
- false, inodesInPath.getLatestSnapshot());
+ false);
newFile = getINodeFile(fsDir, addCloseOp.path);
}
}
@@ -283,7 +282,7 @@ public class FSEditLogLoader {
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime);
- newFile.setModificationTime(addCloseOp.mtime);
+ newFile.setModificationTimeForce(addCloseOp.mtime);
updateBlocks(fsDir, addCloseOp, newFile);
break;
}
@@ -297,8 +296,7 @@ public class FSEditLogLoader {
" clientMachine " + addCloseOp.clientMachine);
}
- final INodesInPath inodesInPath = fsDir.getINodesInPath(addCloseOp.path);
- INodeFile oldFile = toINodeFile(inodesInPath.getINode(0), addCloseOp.path);
+ INodeFile oldFile = getINodeFile(fsDir, addCloseOp.path);
if (oldFile == null) {
throw new IOException("Operation trying to close non-existent file " +
addCloseOp.path);
@@ -306,7 +304,7 @@ public class FSEditLogLoader {
// Update the salient file attributes.
oldFile.setAccessTime(addCloseOp.atime);
- oldFile.setModificationTime(addCloseOp.mtime);
+ oldFile.setModificationTimeForce(addCloseOp.mtime);
updateBlocks(fsDir, addCloseOp, oldFile);
// Now close the file
@@ -324,8 +322,7 @@ public class FSEditLogLoader {
INodeFileUnderConstruction ucFile = (INodeFileUnderConstruction) oldFile;
fsNamesys.leaseManager.removeLeaseWithPrefixPath(addCloseOp.path);
INodeFile newFile = ucFile.convertToInodeFile();
- fsDir.replaceNode(addCloseOp.path, ucFile, newFile,
- inodesInPath.getLatestSnapshot());
+ fsDir.replaceNode(addCloseOp.path, ucFile, newFile);
}
break;
}
@@ -363,8 +360,10 @@ public class FSEditLogLoader {
}
case OP_RENAME_OLD: {
RenameOldOp renameOp = (RenameOldOp)op;
+ HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false);
fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst,
renameOp.timestamp);
+ fsNamesys.unprotectedChangeLease(renameOp.src, renameOp.dst, dinfo);
break;
}
case OP_DELETE: {
@@ -434,8 +433,11 @@ public class FSEditLogLoader {
}
case OP_RENAME: {
RenameOp renameOp = (RenameOp)op;
+
+ HdfsFileStatus dinfo = fsDir.getFileInfo(renameOp.dst, false);
fsDir.unprotectedRenameTo(renameOp.src, renameOp.dst,
renameOp.timestamp, renameOp.options);
+ fsNamesys.unprotectedChangeLease(renameOp.src, renameOp.dst, dinfo);
break;
}
case OP_GET_DELEGATION_TOKEN: {
@@ -510,11 +512,7 @@ public class FSEditLogLoader {
private static INodeFile getINodeFile(FSDirectory fsDir, String path)
throws IOException {
- return toINodeFile(fsDir.getINode(path), path);
- }
-
- private static INodeFile toINodeFile(INode inode, String path)
- throws IOException {
+ INode inode = fsDir.getINode(path);
if (inode != null) {
if (!(inode instanceof INodeFile)) {
throw new IOException("Operation trying to get non-file " + path);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Mon Dec 3 22:43:00 2012
@@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
-import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
@@ -203,7 +202,7 @@ class FSImageFormat {
if (nsQuota != -1 || dsQuota != -1) {
fsDir.rootDir.setQuota(nsQuota, dsQuota);
}
- fsDir.rootDir.cloneModificationTime(root);
+ fsDir.rootDir.setModificationTime(root.getModificationTime());
fsDir.rootDir.clonePermissionStatus(root);
}
@@ -306,7 +305,7 @@ class FSImageFormat {
*/
void addToParent(INodeDirectory parent, INode child) {
// NOTE: This does not update space counts for parents
- if (!parent.addChild(child, false, null)) {
+ if (!parent.addChild(child, false)) {
return;
}
namesystem.dir.cacheName(child);
@@ -389,9 +388,8 @@ class FSImageFormat {
// verify that file exists in namespace
String path = cons.getLocalName();
- final INodesInPath inodesInPath = fsDir.getINodesInPath(path);
- INodeFile oldnode = INodeFile.valueOf(inodesInPath.getINode(0), path);
- fsDir.replaceNode(path, oldnode, cons, inodesInPath.getLatestSnapshot());
+ INodeFile oldnode = INodeFile.valueOf(fsDir.getINode(path), path);
+ fsDir.replaceNode(path, oldnode, cons);
namesystem.leaseManager.addLease(cons.getClientName(), path);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Dec 3 22:43:00 2012
@@ -121,7 +121,6 @@ import org.apache.hadoop.fs.UnresolvedLi
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
@@ -166,6 +165,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
@@ -173,9 +173,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithLink;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
@@ -1038,8 +1036,7 @@ public class FSNamesystem implements Nam
long totalInodes = this.dir.totalInodes();
long totalBlocks = this.getBlocksTotal();
out.println(totalInodes + " files and directories, " + totalBlocks
- + " blocks = " + (totalInodes + totalBlocks)
- + " total filesystem objects");
+ + " blocks = " + (totalInodes + totalBlocks) + " total");
blockManager.metaSave(out);
}
@@ -1823,9 +1820,7 @@ public class FSNamesystem implements Nam
try {
blockManager.verifyReplication(src, replication, clientMachine);
boolean create = flag.contains(CreateFlag.CREATE);
-
- final INodesInPath inodesInPath = dir.getINodesInPath(src);
- final INode myFile = inodesInPath.getINode(0);
+ final INode myFile = dir.getINode(src);
if (myFile == null) {
if (!create) {
throw new FileNotFoundException("failed to overwrite or append to non-existent file "
@@ -1852,8 +1847,8 @@ public class FSNamesystem implements Nam
if (append && myFile != null) {
final INodeFile f = INodeFile.valueOf(myFile, src);
- return prepareFileForWrite(src, f, holder, clientMachine, clientNode,
- true, inodesInPath.getLatestSnapshot());
+ return prepareFileForWrite(
+ src, f, holder, clientMachine, clientNode, true);
} else {
// Now we can add the name to the filesystem. This file has no
// blocks associated with it.
@@ -1901,7 +1896,7 @@ public class FSNamesystem implements Nam
*/
LocatedBlock prepareFileForWrite(String src, INodeFile file,
String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
- boolean writeToEditLog, Snapshot latestSnapshot) throws IOException {
+ boolean writeToEditLog) throws IOException {
//TODO SNAPSHOT: INodeFileUnderConstruction with link
INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
file.getLocalNameBytes(),
@@ -1913,7 +1908,7 @@ public class FSNamesystem implements Nam
leaseHolder,
clientMachine,
clientNode);
- dir.replaceNode(src, file, cons, latestSnapshot);
+ dir.replaceNode(src, file, cons);
leaseManager.addLease(cons.getClientName(), src);
LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons);
@@ -2163,8 +2158,7 @@ public class FSNamesystem implements Nam
// have we exceeded the configured limit of fs objects.
checkFsObjectLimit();
- final INodeFileUnderConstruction pendingFile = checkLease(
- src, clientName, dir.getINode(src));
+ INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
BlockInfo lastBlockInFile = pendingFile.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
@@ -2300,8 +2294,7 @@ public class FSNamesystem implements Nam
}
//check lease
- final INodeFileUnderConstruction file = checkLease(
- src, clientName, dir.getINode(src));
+ final INodeFileUnderConstruction file = checkLease(src, clientName);
clientnode = file.getClientNode();
preferredblocksize = file.getPreferredBlockSize();
@@ -2347,9 +2340,7 @@ public class FSNamesystem implements Nam
throw new SafeModeException("Cannot abandon block " + b +
" for fle" + src, safeMode);
}
- final INodesInPath inodesInPath = checkLease(src, holder);
- final INodeFileUnderConstruction file
- = (INodeFileUnderConstruction)inodesInPath.getINode(0);
+ INodeFileUnderConstruction file = checkLease(src, holder);
dir.removeBlock(src, file, ExtendedBlock.getLocalBlock(b));
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: "
@@ -2366,13 +2357,11 @@ public class FSNamesystem implements Nam
return true;
}
- /** make sure that we still have the lease on this file. */
- private INodesInPath checkLease(String src, String holder)
+ // make sure that we still have the lease on this file.
+ private INodeFileUnderConstruction checkLease(String src, String holder)
throws LeaseExpiredException, UnresolvedLinkException {
assert hasReadOrWriteLock();
- final INodesInPath inodesInPath = dir.getINodesInPath(src);
- checkLease(src, holder, inodesInPath.getINode(0));
- return inodesInPath;
+ return checkLease(src, holder, dir.getINode(src));
}
private INodeFileUnderConstruction checkLease(String src, String holder,
@@ -2435,11 +2424,9 @@ public class FSNamesystem implements Nam
throw new SafeModeException("Cannot complete file " + src, safeMode);
}
- final INodesInPath inodesInPath;
- final INodeFileUnderConstruction pendingFile;
+ INodeFileUnderConstruction pendingFile;
try {
- inodesInPath = checkLease(src, holder);
- pendingFile = (INodeFileUnderConstruction)inodesInPath.getINode(0);
+ pendingFile = checkLease(src, holder);
} catch (LeaseExpiredException lee) {
final INode inode = dir.getINode(src);
if (inode != null && inode instanceof INodeFile && !inode.isUnderConstruction()) {
@@ -2467,8 +2454,7 @@ public class FSNamesystem implements Nam
return false;
}
- finalizeINodeFileUnderConstruction(src, pendingFile,
- inodesInPath.getLatestSnapshot());
+ finalizeINodeFileUnderConstruction(src, pendingFile);
NameNode.stateChangeLog.info("DIR* completeFile: " + src + " is closed by "
+ holder);
@@ -2609,15 +2595,15 @@ public class FSNamesystem implements Nam
if (isPermissionEnabled) {
//We should not be doing this. This is move() not renameTo().
//but for now,
- //NOTE: yes, this is bad! it's assuming much lower level behavior
- // of rewriting the dst
String actualdst = dir.isDir(dst)?
dst + Path.SEPARATOR + new Path(src).getName(): dst;
checkParentAccess(src, FsAction.WRITE);
checkAncestorAccess(actualdst, FsAction.WRITE);
}
+ HdfsFileStatus dinfo = dir.getFileInfo(dst, false);
if (dir.renameTo(src, dst)) {
+ unprotectedChangeLease(src, dst, dinfo); // update lease with new filename
return true;
}
return false;
@@ -2668,7 +2654,9 @@ public class FSNamesystem implements Nam
checkAncestorAccess(dst, FsAction.WRITE);
}
+ HdfsFileStatus dinfo = dir.getFileInfo(dst, false);
dir.renameTo(src, dst, options);
+ unprotectedChangeLease(src, dst, dinfo); // update lease with new filename
}
/**
@@ -3048,8 +3036,7 @@ public class FSNamesystem implements Nam
if (isInSafeMode()) {
throw new SafeModeException("Cannot fsync file " + src, safeMode);
}
- final INodeFileUnderConstruction pendingFile = checkLease(
- src, clientName, dir.getINode(src));
+ INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
if (lastBlockLength > 0) {
pendingFile.updateLengthOfLastBlock(lastBlockLength);
}
@@ -3081,9 +3068,8 @@ public class FSNamesystem implements Nam
assert !isInSafeMode();
assert hasWriteLock();
- final INodesInPath inodesInPath = dir.getINodesInPath(src);
final INodeFileUnderConstruction pendingFile
- = INodeFileUnderConstruction.valueOf(inodesInPath.getINode(0), src);
+ = INodeFileUnderConstruction.valueOf(dir.getINode(src), src);
int nrBlocks = pendingFile.numBlocks();
BlockInfo[] blocks = pendingFile.getBlocks();
@@ -3100,8 +3086,7 @@ public class FSNamesystem implements Nam
// If there are no incomplete blocks associated with this file,
// then reap lease immediately and close the file.
if(nrCompleteBlocks == nrBlocks) {
- finalizeINodeFileUnderConstruction(src, pendingFile,
- inodesInPath.getLatestSnapshot());
+ finalizeINodeFileUnderConstruction(src, pendingFile);
NameNode.stateChangeLog.warn("BLOCK*"
+ " internalReleaseLease: All existing blocks are COMPLETE,"
+ " lease removed, file closed.");
@@ -3149,8 +3134,7 @@ public class FSNamesystem implements Nam
// Close file if committed blocks are minimally replicated
if(penultimateBlockMinReplication &&
blockManager.checkMinReplication(lastBlock)) {
- finalizeINodeFileUnderConstruction(src, pendingFile,
- inodesInPath.getLatestSnapshot());
+ finalizeINodeFileUnderConstruction(src, pendingFile);
NameNode.stateChangeLog.warn("BLOCK*"
+ " internalReleaseLease: Committed blocks are minimally replicated,"
+ " lease removed, file closed.");
@@ -3228,7 +3212,7 @@ public class FSNamesystem implements Nam
}
private void finalizeINodeFileUnderConstruction(String src,
- INodeFileUnderConstruction pendingFile, Snapshot latestSnapshot)
+ INodeFileUnderConstruction pendingFile)
throws IOException, UnresolvedLinkException {
assert hasWriteLock();
leaseManager.removeLease(pendingFile.getClientName(), src);
@@ -3236,7 +3220,7 @@ public class FSNamesystem implements Nam
// The file is no longer pending.
// Create permanent INode, update blocks
INodeFile newFile = pendingFile.convertToInodeFile();
- dir.replaceNode(src, pendingFile, newFile, latestSnapshot);
+ dir.replaceNode(src, pendingFile, newFile);
// close file and persist block allocations for this file
dir.closeFile(src, newFile);
@@ -3328,8 +3312,7 @@ public class FSNamesystem implements Nam
commitOrCompleteLastBlock(pendingFile, storedBlock);
//remove lease, close file
- finalizeINodeFileUnderConstruction(src, pendingFile,
- INodeDirectorySnapshottable.findLatestSnapshot(pendingFile));
+ finalizeINodeFileUnderConstruction(src, pendingFile);
} else {
// If this commit does not want to close the file, persist blocks
dir.persistBlocks(src, pendingFile);
@@ -3500,9 +3483,9 @@ public class FSNamesystem implements Nam
private NNHAStatusHeartbeat createHaStatusHeartbeat() {
HAState state = haContext.getState();
NNHAStatusHeartbeat.State hbState;
- if (state.getServiceState() == HAServiceState.ACTIVE) {
+ if (state instanceof ActiveState) {
hbState = NNHAStatusHeartbeat.State.ACTIVE;
- } else if (state.getServiceState() == HAServiceState.STANDBY) {
+ } else if (state instanceof StandbyState) {
hbState = NNHAStatusHeartbeat.State.STANDBY;
} else {
throw new AssertionError("Invalid state: " + state.getClass());
@@ -4956,9 +4939,31 @@ public class FSNamesystem implements Nam
// rename was successful. If any part of the renamed subtree had
// files that were being written to, update with new filename.
- void unprotectedChangeLease(String src, String dst) {
+ void unprotectedChangeLease(String src, String dst, HdfsFileStatus dinfo) {
+ String overwrite;
+ String replaceBy;
assert hasWriteLock();
- leaseManager.changeLease(src, dst);
+
+ boolean destinationExisted = true;
+ if (dinfo == null) {
+ destinationExisted = false;
+ }
+
+ if (destinationExisted && dinfo.isDir()) {
+ Path spath = new Path(src);
+ Path parent = spath.getParent();
+ if (parent.isRoot()) {
+ overwrite = parent.toString();
+ } else {
+ overwrite = parent.toString() + Path.SEPARATOR;
+ }
+ replaceBy = dst + Path.SEPARATOR;
+ } else {
+ overwrite = src;
+ replaceBy = dst;
+ }
+
+ leaseManager.changeLease(src, dst, overwrite, replaceBy);
}
/**
@@ -4969,13 +4974,19 @@ public class FSNamesystem implements Nam
// lock on our behalf. If we took the read lock here, we could block
// for fairness if a writer is waiting on the lock.
synchronized (leaseManager) {
- Map<String, INodeFileUnderConstruction> nodes =
- leaseManager.getINodesUnderConstruction();
- out.writeInt(nodes.size()); // write the size
- for (Map.Entry<String, INodeFileUnderConstruction> entry
- : nodes.entrySet()) {
- FSImageSerialization.writeINodeUnderConstruction(
- out, entry.getValue(), entry.getKey());
+ out.writeInt(leaseManager.countPath()); // write the size
+
+ for (Lease lease : leaseManager.getSortedLeases()) {
+ for(String path : lease.getPaths()) {
+ // verify that path exists in namespace
+ final INodeFileUnderConstruction cons;
+ try {
+ cons = INodeFileUnderConstruction.valueOf(dir.getINode(path), path);
+ } catch (UnresolvedLinkException e) {
+ throw new AssertionError("Lease files should reside on this FS");
+ }
+ FSImageSerialization.writeINodeUnderConstruction(out, cons, path);
+ }
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Mon Dec 3 22:43:00 2012
@@ -33,7 +33,6 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.util.StringUtils;
@@ -47,33 +46,8 @@ import com.google.common.primitives.Sign
*/
@InterfaceAudience.Private
public abstract class INode implements Comparable<byte[]> {
- /** A dummy INode which can be used as a probe object. */
- public static final INode DUMMY = new INode() {
- @Override
- int collectSubtreeBlocksAndClear(BlocksMapUpdateInfo info) {
- throw new UnsupportedOperationException();
- }
- @Override
- long[] computeContentSummary(long[] summary) {
- throw new UnsupportedOperationException();
- }
- @Override
- DirCounts spaceConsumedInTree(DirCounts counts) {
- throw new UnsupportedOperationException();
- }
- };
static final ReadOnlyList<INode> EMPTY_READ_ONLY_LIST
= ReadOnlyList.Util.emptyList();
- /**
- * Assert that the snapshot parameter must be null since
- * this class only take care current state.
- * Subclasses should override the methods for handling the snapshot states.
- */
- static void assertNull(Snapshot snapshot) {
- if (snapshot != null) {
- throw new AssertionError("snapshot is not null: " + snapshot);
- }
- }
/** Wrapper of two counters for namespace consumed and diskspace consumed. */
@@ -146,12 +120,9 @@ public abstract class INode implements C
* should not modify it.
*/
private long permission = 0L;
- INodeDirectory parent = null;
- private long modificationTime = 0L;
- private long accessTime = 0L;
-
- /** For creating the a {@link #DUMMY} object. */
- private INode() {}
+ protected INodeDirectory parent = null;
+ protected long modificationTime = 0L;
+ protected long accessTime = 0L;
private INode(byte[] name, long permission, INodeDirectory parent,
long modificationTime, long accessTime) {
@@ -178,8 +149,8 @@ public abstract class INode implements C
/** @param other Other node to be copied */
INode(INode other) {
- this(other.name, other.permission, other.parent,
- other.modificationTime, other.accessTime);
+ this(other.getLocalNameBytes(), other.permission, other.getParent(),
+ other.getModificationTime(), other.getAccessTime());
}
/**
@@ -319,13 +290,13 @@ public abstract class INode implements C
* Set local file name
*/
public void setLocalName(String name) {
- setLocalName(DFSUtil.string2Bytes(name));
+ this.name = DFSUtil.string2Bytes(name);
}
/**
* Set local file name
*/
- public void setLocalName(byte[] name) {
+ void setLocalName(byte[] name) {
this.name = name;
}
@@ -345,7 +316,7 @@ public abstract class INode implements C
* Get parent directory
* @return parent INode
*/
- public INodeDirectory getParent() {
+ INodeDirectory getParent() {
return this.parent;
}
@@ -365,21 +336,17 @@ public abstract class INode implements C
/**
* Set last modification time of inode.
*/
- public void updateModificationTime(long modtime) {
+ public void setModificationTime(long modtime) {
assert isDirectory();
if (this.modificationTime <= modtime) {
- setModificationTime(modtime);
+ this.modificationTime = modtime;
}
}
- void cloneModificationTime(INode that) {
- this.modificationTime = that.modificationTime;
- }
-
/**
* Always set the last modification time of inode.
*/
- void setModificationTime(long modtime) {
+ void setModificationTimeForce(long modtime) {
this.modificationTime = modtime;
}
@@ -464,11 +431,11 @@ public abstract class INode implements C
return buf.toString();
}
- public boolean removeNode(Snapshot latestSnapshot) {
+ public boolean removeNode() {
if (parent == null) {
return false;
} else {
- parent.removeChild(this, latestSnapshot);
+ parent.removeChild(this);
parent = null;
return true;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Mon Dec 3 22:43:00 2012
@@ -54,7 +54,7 @@ public class INodeDirectory extends INod
}
return (INodeDirectory)inode;
}
-
+
protected static final int DEFAULT_FILES_PER_DIRECTORY = 5;
final static String ROOT_NAME = "";
@@ -99,56 +99,32 @@ public class INodeDirectory extends INod
}
}
- public int searchChildren(INode inode) {
+ private int searchChildren(INode inode) {
return Collections.binarySearch(children, inode.getLocalNameBytes());
}
- public int searchChildrenForExistingINode(INode inode) {
- final int i = searchChildren(inode);
- if (i < 0) {
- throw new AssertionError("Child not found: inode=" + inode);
- }
- return i;
- }
-
- public INode removeChild(INode node, Snapshot latestSnapshot) {
+ INode removeChild(INode node) {
assertChildrenNonNull();
-
- if (latestSnapshot != null) {
- final INodeDirectoryWithSnapshot dir
- = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
- return dir.removeChild(node, latestSnapshot);
- }
-
final int i = searchChildren(node);
return i >= 0? children.remove(i): null;
}
+
/** Replace a child that has the same name as newChild by newChild.
*
* @param newChild Child node to be added
*/
- public INode replaceChild(INodeDirectory newChild, Snapshot latestSnapshot) {
+ void replaceChild(INode newChild) {
assertChildrenNonNull();
- if (latestSnapshot != null) {
- final INodeDirectoryWithSnapshot dir
- = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
- return dir.replaceChild(newChild, latestSnapshot);
- }
-
- // find the old child and replace it
- final int low = searchChildrenForExistingINode(newChild);
- final INode oldChild = children.set(low, newChild);
- // set the parent of the children of the child.
- for(INode i : newChild.getChildrenList(null)) {
- i.parent = newChild;
+ final int low = searchChildren(newChild);
+ if (low>=0) { // an old child exists so replace by the newChild
+ children.set(low, newChild);
+ } else {
+ throw new IllegalArgumentException("No child exists to be replaced");
}
- return oldChild;
}
- public INode getChild(byte[] name, Snapshot snapshot) {
- assertNull(snapshot);
-
+ private INode getChild(byte[] name, Snapshot snapshot) {
final ReadOnlyList<INode> c = getChildrenList(snapshot);
final int i = ReadOnlyList.Util.binarySearch(c, name);
return i < 0? null: c.get(i);
@@ -385,14 +361,7 @@ public class INodeDirectory extends INod
* @return false if the child with this name already exists;
* otherwise, return true;
*/
- public boolean addChild(final INode node, final boolean setModTime,
- Snapshot latestSnapshot) {
- if (latestSnapshot != null) {
- final INodeDirectoryWithSnapshot dir
- = INodeDirectoryWithSnapshot.replaceDir(this, latestSnapshot);
- return dir.addChild(node, setModTime, latestSnapshot);
- }
-
+ public boolean addChild(final INode node, final boolean setModTime) {
if (children == null) {
children = new ArrayList<INode>(DEFAULT_FILES_PER_DIRECTORY);
}
@@ -403,9 +372,8 @@ public class INodeDirectory extends INod
node.parent = this;
children.add(-low - 1, node);
// update modification time of the parent directory
- if (setModTime) {
- updateModificationTime(node.getModificationTime());
- }
+ if (setModTime)
+ setModificationTime(node.getModificationTime());
if (node.getGroupName() == null) {
node.setGroup(getGroupName());
}
@@ -432,28 +400,20 @@ public class INodeDirectory extends INod
}
newNode.setLocalName(pathComponents[pathComponents.length - 1]);
// insert into the parent children list
- INodesInPath inodes = getExistingPathINodes(pathComponents, 2, false);
- INodeDirectory parent = INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
- return parent.addChild(newNode, true, inodes.getLatestSnapshot());
+ INodeDirectory parent = getParent(pathComponents);
+ return parent.addChild(newNode, true);
}
INodeDirectory getParent(byte[][] pathComponents
) throws FileNotFoundException, PathIsNotDirectoryException,
UnresolvedLinkException {
- return (INodeDirectory)getParentINodesInPath(pathComponents).getINode(0);
- }
-
- INodesInPath getParentINodesInPath(byte[][] pathComponents
- ) throws FileNotFoundException, PathIsNotDirectoryException,
- UnresolvedLinkException {
if (pathComponents.length < 2) // add root
return null;
// Gets the parent INode
INodesInPath inodes = getExistingPathINodes(pathComponents, 2, false);
- INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
- return inodes;
+ return INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
}
-
+
@Override
DirCounts spaceConsumedInTree(DirCounts counts) {
counts.nsCount += 1;
@@ -502,11 +462,10 @@ public class INodeDirectory extends INod
* Note that the returned list is never null.
*/
public ReadOnlyList<INode> getChildrenList(final Snapshot snapshot) {
- assertNull(snapshot);
+ //TODO: use snapshot to select children list
return children == null ? EMPTY_READ_ONLY_LIST
: ReadOnlyList.Util.asReadOnlyList(children);
}
-
/** Set the children list. */
public void setChildren(List<INode> children) {
this.children = children;
@@ -531,7 +490,7 @@ public class INodeDirectory extends INod
* {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)}.
* Contains INodes information resolved from a given path.
*/
- public static class INodesInPath {
+ static class INodesInPath {
/**
* Array with the specified number of INodes resolved for a given path.
*/
@@ -611,7 +570,7 @@ public class INodeDirectory extends INod
}
/** @return the i-th inode. */
- public INode getINode(int i) {
+ INode getINode(int i) {
return inodes[i];
}
@@ -717,13 +676,4 @@ public class INodeDirectory extends INod
}
}
}
-
- /**
- * Get last modification time of inode.
- * @return access time
- */
- public long getModificationTime(Snapshot snapshot) {
- assertNull(snapshot);
- return getModificationTime();
- }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Mon Dec 3 22:43:00 2012
@@ -89,22 +89,16 @@ public class INodeFile extends INode imp
INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
short replication, long modificationTime,
long atime, long preferredBlockSize) {
- this(null, permissions, modificationTime, atime, blklist, replication,
- preferredBlockSize);
- }
-
- INodeFile(byte[] name, PermissionStatus permissions, long mtime, long atime,
- BlockInfo[] blklist, short replication, long preferredBlockSize) {
- super(name, permissions, null, mtime, atime);
+ super(permissions, modificationTime, atime);
header = HeaderFormat.combineReplication(header, replication);
header = HeaderFormat.combinePreferredBlockSize(header, preferredBlockSize);
this.blocks = blklist;
}
- protected INodeFile(INodeFile that) {
- super(that);
- this.header = that.header;
- this.blocks = that.blocks;
+ protected INodeFile(INodeFile f) {
+ this(f.getPermissionStatus(), f.getBlocks(), f.getFileReplication(),
+ f.getModificationTime(), f.getAccessTime(), f.getPreferredBlockSize());
+ this.setLocalName(f.getLocalNameBytes());
}
/** @return true unconditionally. */
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java Mon Dec 3 22:43:00 2012
@@ -72,8 +72,9 @@ public class INodeFileUnderConstruction
String clientName,
String clientMachine,
DatanodeDescriptor clientNode) {
- super(name, perm, modificationTime, modificationTime,
- blocks, blockReplication, preferredBlockSize);
+ super(perm, blocks, blockReplication, modificationTime, modificationTime,
+ preferredBlockSize);
+ setLocalName(name);
this.clientName = clientName;
this.clientMachine = clientMachine;
this.clientNode = clientNode;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Mon Dec 3 22:43:00 2012
@@ -331,19 +331,22 @@ public class LeaseManager {
}
}
- synchronized void changeLease(String src, String dst) {
+ synchronized void changeLease(String src, String dst,
+ String overwrite, String replaceBy) {
if (LOG.isDebugEnabled()) {
LOG.debug(getClass().getSimpleName() + ".changelease: " +
- " src=" + src + ", dest=" + dst);
+ " src=" + src + ", dest=" + dst +
+ ", overwrite=" + overwrite +
+ ", replaceBy=" + replaceBy);
}
- final int len = src.length();
+ final int len = overwrite.length();
for(Map.Entry<String, Lease> entry
: findLeaseWithPrefixPath(src, sortedLeasesByPath).entrySet()) {
final String oldpath = entry.getKey();
final Lease lease = entry.getValue();
- // replace stem of src with new destination
- final String newpath = dst + oldpath.substring(len);
+ //overwrite must be a prefix of oldpath
+ final String newpath = replaceBy + oldpath.substring(len);
if (LOG.isDebugEnabled()) {
LOG.debug("changeLease: replacing " + oldpath + " with " + newpath);
}
@@ -426,26 +429,6 @@ public class LeaseManager {
}
}
- /**
- * Get the list of inodes corresponding to valid leases.
- * @return list of inodes
- * @throws UnresolvedLinkException
- */
- Map<String, INodeFileUnderConstruction> getINodesUnderConstruction() {
- Map<String, INodeFileUnderConstruction> inodes =
- new TreeMap<String, INodeFileUnderConstruction>();
- for (String p : sortedLeasesByPath.keySet()) {
- // verify that path exists in namespace
- try {
- INode node = fsnamesystem.dir.getINode(p);
- inodes.put(p, INodeFileUnderConstruction.valueOf(node, p));
- } catch (IOException ioe) {
- LOG.error(ioe);
- }
- }
- return inodes;
- }
-
/** Check the leases beginning from the oldest.
* @return true is sync is needed.
*/
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Mon Dec 3 22:43:00 2012
@@ -598,7 +598,11 @@ public class NameNode {
String nsId = getNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
this.haEnabled = HAUtil.isHAEnabled(conf, nsId);
- state = createHAState();
+ if (!haEnabled) {
+ state = ACTIVE_STATE;
+ } else {
+ state = STANDBY_STATE;
+ }
this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf);
this.haContext = createHAContext();
try {
@@ -615,10 +619,6 @@ public class NameNode {
}
}
- protected HAState createHAState() {
- return !haEnabled ? ACTIVE_STATE : STANDBY_STATE;
- }
-
protected HAContext createHAContext() {
return new NameNodeHAContext();
}
@@ -1298,7 +1298,7 @@ public class NameNode {
* before exit.
* @throws ExitException thrown only for testing.
*/
- protected synchronized void doImmediateShutdown(Throwable t)
+ private synchronized void doImmediateShutdown(Throwable t)
throws ExitException {
String message = "Error encountered requiring NN shutdown. " +
"Shutting down immediately.";
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Mon Dec 3 22:43:00 2012
@@ -102,7 +102,7 @@ class NamenodeJspHelper {
long usedNonHeap = (totalNonHeap * 100) / commitedNonHeap;
String str = "<div>" + inodes + " files and directories, " + blocks + " blocks = "
- + (inodes + blocks) + " total filesystem objects";
+ + (inodes + blocks) + " total";
if (maxobjects != 0) {
long pct = ((inodes + blocks) * 100) / maxobjects;
str += " / " + maxobjects + " (" + pct + "%)";
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Mon Dec 3 22:43:00 2012
@@ -65,19 +65,6 @@ public class INodeDirectorySnapshottable
}
return (INodeDirectorySnapshottable)dir;
}
-
- public static Snapshot findLatestSnapshot(INode inode) {
- Snapshot latest = null;
- for(; inode != null; inode = inode.getParent()) {
- if (inode instanceof INodeDirectorySnapshottable) {
- final Snapshot s = ((INodeDirectorySnapshottable)inode).getLastSnapshot();
- if (Snapshot.ID_COMPARATOR.compare(latest, s) < 0) {
- latest = s;
- }
- }
- }
- return latest;
- }
/** Snapshots of this directory in ascending order of snapshot id. */
private final List<Snapshot> snapshots = new ArrayList<Snapshot>();
@@ -209,8 +196,8 @@ public class INodeDirectorySnapshottable
//set modification time
final long timestamp = Time.now();
- s.getRoot().updateModificationTime(timestamp);
- updateModificationTime(timestamp);
+ s.getRoot().setModificationTime(timestamp);
+ setModificationTime(timestamp);
return s;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java Mon Dec 3 22:43:00 2012
@@ -19,15 +19,10 @@ package org.apache.hadoop.hdfs.server.na
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
-import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
-import org.apache.hadoop.hdfs.util.ReadOnlyList;
-
-import com.google.common.base.Preconditions;
/** The directory with snapshots. */
public class INodeDirectoryWithSnapshot extends INodeDirectory {
@@ -255,223 +250,9 @@ public class INodeDirectoryWithSnapshot
+ "\n deleted=" + toString(deleted);
}
}
-
- private class SnapshotDiff implements Comparable<Snapshot> {
- /** The snapshot will obtain after applied this diff. */
- final Snapshot snapshot;
- /** The size of the children list which is never changed. */
- final int size;
- /**
- * Posterior diff is the diff happened after this diff.
- * The posterior diff should be first applied to obtain the posterior
- * snapshot and then apply this diff in order to obtain this snapshot.
- * If the posterior diff is null, the posterior state is the current state.
- */
- private SnapshotDiff posteriorDiff;
- /** The data of this diff. */
- private final Diff diff = new Diff();
- /** The snapshot version of the inode. */
- private INode snapshotINode;
-
- SnapshotDiff(Snapshot snapshot, int size) {
- if (size < 0) {
- throw new HadoopIllegalArgumentException("size = " + size + " < 0");
- }
- this.snapshot = snapshot;
- this.size = size;
- }
-
- @Override
- public int compareTo(final Snapshot that_snapshot) {
- return Snapshot.ID_COMPARATOR.compare(this.snapshot, that_snapshot);
- }
-
- /**
- * @return The children list of a directory in a snapshot.
- * Since the snapshot is read-only, the logical view of the list is
- * never changed although the internal data structure may mutate.
- */
- ReadOnlyList<INode> getChildrenList() {
- return new ReadOnlyList<INode>() {
- private List<INode> children = null;
-
- private List<INode> initChildren() {
- if (children == null) {
- final ReadOnlyList<INode> posterior = posteriorDiff != null?
- posteriorDiff.getChildrenList()
- : INodeDirectoryWithSnapshot.this.getChildrenList(null);
- children = diff.apply2Current(ReadOnlyList.Util.asList(posterior));
- }
- return children;
- }
-
- @Override
- public Iterator<INode> iterator() {
- return initChildren().iterator();
- }
-
- @Override
- public boolean isEmpty() {
- return size == 0;
- }
-
- @Override
- public int size() {
- return size;
- }
-
- @Override
- public INode get(int i) {
- return initChildren().get(i);
- }
- };
- }
-
- INode getChild(byte[] name) {
- final INode i = diff.accessPrevious(name, INode.DUMMY);
- if (i != INode.DUMMY) {
- // this diff is able to find it
- return i;
- } else {
- // should return the posterior INode.
- return posteriorDiff != null? posteriorDiff.getChild(name)
- : INodeDirectoryWithSnapshot.this.getChild(name, null);
- }
- }
- }
-
- /** Replace the given directory to an {@link INodeDirectoryWithSnapshot}. */
- public static INodeDirectoryWithSnapshot replaceDir(INodeDirectory oldDir,
- Snapshot latestSnapshot) {
- Preconditions.checkArgument(!(oldDir instanceof INodeDirectoryWithSnapshot),
- "oldDir is already an INodeDirectoryWithSnapshot, oldDir=%s", oldDir);
-
- final INodeDirectory parent = oldDir.getParent();
- Preconditions.checkArgument(parent != null,
- "parent is null, oldDir=%s", oldDir);
-
- final INodeDirectoryWithSnapshot newDir = new INodeDirectoryWithSnapshot(
- oldDir, latestSnapshot);
- parent.replaceChild(newDir, null);
- return newDir;
- }
-
- /** Diff list sorted by snapshot IDs, i.e. in chronological order. */
- private final List<SnapshotDiff> diffs = new ArrayList<SnapshotDiff>();
- INodeDirectoryWithSnapshot(INodeDirectory that, Snapshot s) {
- super(that);
-
- // add a diff for the snapshot
- addSnapshotDiff(s, that.getChildrenList(null).size());
- }
-
- INodeDirectoryWithSnapshot(String name, INodeDirectory dir, Snapshot s) {
- this(dir, s);
- setLocalName(name);
- setParent(dir);
- }
-
- SnapshotDiff addSnapshotDiff(Snapshot snapshot, int childrenSize) {
- final SnapshotDiff d = new SnapshotDiff(snapshot, childrenSize);
- diffs.add(d);
- return d;
- }
-
- /**
- * Check if the latest snapshot diff exist. If not, add it.
- * @return the latest snapshot diff, which is never null.
- */
- private SnapshotDiff checkAndAddLatestSnapshotDiff(Snapshot latest) {
- final SnapshotDiff last = getLastSnapshotDiff();
- if (last != null && last.snapshot.equals(latest)) {
- return last;
- }
-
- final int size = getChildrenList(null).size();
- final SnapshotDiff d = addSnapshotDiff(latest, size);
- if (last != null) {
- last.posteriorDiff = d;
- }
- return d;
- }
-
- Diff getLatestDiff(Snapshot latest) {
- return checkAndAddLatestSnapshotDiff(latest).diff;
- }
-
- /**
- * @return the diff corresponding to the snapshot.
- * When the diff is not found, it means that the current state and
- * the snapshot state are the same.
- */
- SnapshotDiff getSnapshotDiff(Snapshot snapshot) {
- if (snapshot == null) {
- return null;
- }
- final int i = Collections.binarySearch(diffs, snapshot);
- if (i >= 0) {
- // exact match
- return diffs.get(i);
- } else {
- // Exact match not found means that there were no changes between
- // given snapshot and the next state so that the diff for the given
- // snapshot is not recorded. Thus, use the next state.
- final int j = -i - 1;
- return j < diffs.size()? diffs.get(j): null;
- }
- }
-
- SnapshotDiff getLastSnapshotDiff() {
- return diffs.get(diffs.size() - 1);
- }
-
- @Override
- public ReadOnlyList<INode> getChildrenList(Snapshot snapshot) {
- final SnapshotDiff diff = getSnapshotDiff(snapshot);
- if (diff != null) {
- return diff.getChildrenList();
- }
- return super.getChildrenList(null);
- }
-
- @Override
- public INode getChild(byte[] name, Snapshot snapshot) {
- final SnapshotDiff diff = getSnapshotDiff(snapshot);
- if (diff != null) {
- return diff.getChild(name);
- }
- return super.getChild(name, null);
- }
-
- @Override
- public boolean addChild(INode inode, boolean setModTime,
- Snapshot latestSnapshot) {
- getLatestDiff(latestSnapshot).create(inode);
- return super.addChild(inode, setModTime, null);
- }
-
- @Override
- public INode removeChild(INode inode, Snapshot latestSnapshot) {
- getLatestDiff(latestSnapshot).delete(inode);
- return super.removeChild(inode, null);
- }
-
- @Override
- public INode replaceChild(INodeDirectory newChild, Snapshot latestSnapshot) {
- final INode oldChild = super.replaceChild(newChild, null);
- final Diff diff = getLatestDiff(latestSnapshot);
- diff.delete(oldChild);
- diff.create(newChild);
- return oldChild;
- }
-
- @Override
- public long getModificationTime(Snapshot snapshot) {
- final SnapshotDiff diff = getSnapshotDiff(snapshot);
- if (diff != null) {
- return diff.snapshotINode.getModificationTime();
- }
- return getModificationTime();
+ INodeDirectoryWithSnapshot(String name, INodeDirectory dir) {
+ super(name, dir.getPermissionStatus());
+ parent = dir;
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithLink.java Mon Dec 3 22:43:00 2012
@@ -109,7 +109,7 @@ public class INodeFileWithLink extends I
this.setFileReplication(maxReplication);
this.next = null;
// clear parent
- setParent(null);
+ parent = null;
}
return 1;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java Mon Dec 3 22:43:00 2012
@@ -44,7 +44,7 @@ public class Snapshot implements Compara
Snapshot(int id, String name, INodeDirectorySnapshottable dir) {
this.id = id;
- this.root = new INodeDirectoryWithSnapshot(name, dir, this);
+ this.root = new INodeDirectoryWithSnapshot(name, dir);
}
/** @return the root directory of the snapshot. */
@@ -58,21 +58,6 @@ public class Snapshot implements Compara
}
@Override
- public boolean equals(Object that) {
- if (this == that) {
- return true;
- } else if (that == null || !(that instanceof Snapshot)) {
- return false;
- }
- return this.id == ((Snapshot)that).id;
- }
-
- @Override
- public int hashCode() {
- return id;
- }
-
- @Override
public String toString() {
return getClass().getSimpleName() + ":" + root.getLocalName();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Mon Dec 3 22:43:00 2012
@@ -24,8 +24,12 @@ import java.util.concurrent.atomic.Atomi
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
-import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.INodesInPath;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction;
+import org.apache.hadoop.hdfs.server.namenode.INodeSymlink;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
/**
* Manage snapshottable directories and their snapshots.
@@ -64,9 +68,7 @@ public class SnapshotManager implements
*/
public void setSnapshottable(final String path, final int snapshotQuota
) throws IOException {
- final INodesInPath inodesInPath = fsdir.getINodesInPath(path);
- final INodeDirectory d = INodeDirectory.valueOf(
- inodesInPath.getINode(0), path);
+ final INodeDirectory d = INodeDirectory.valueOf(fsdir.getINode(path), path);
if (d.isSnapshottable()) {
//The directory is already a snapshottable directory.
((INodeDirectorySnapshottable)d).setSnapshotQuota(snapshotQuota);
@@ -75,7 +77,7 @@ public class SnapshotManager implements
final INodeDirectorySnapshottable s
= INodeDirectorySnapshottable.newInstance(d, snapshotQuota);
- fsdir.replaceINodeDirectory(path, d, s, inodesInPath.getLatestSnapshot());
+ fsdir.replaceINodeDirectory(path, d, s);
snapshottables.add(s);
numSnapshottableDirs.getAndIncrement();
@@ -88,16 +90,15 @@ public class SnapshotManager implements
*/
public void resetSnapshottable(final String path
) throws IOException {
- final INodesInPath inodesInPath = fsdir.getINodesInPath(path);
final INodeDirectorySnapshottable s = INodeDirectorySnapshottable.valueOf(
- inodesInPath.getINode(0), path);
+ fsdir.getINode(path), path);
if (s.getNumSnapshots() > 0) {
throw new SnapshotException("The directory " + path + " has snapshot(s). "
+ "Please redo the operation after removing all the snapshots.");
}
final INodeDirectory d = new INodeDirectory(s);
- fsdir.replaceINodeDirectory(path, s, d, inodesInPath.getLatestSnapshot());
+ fsdir.replaceINodeDirectory(path, s, d);
snapshottables.remove(s);
numSnapshottableDirs.getAndDecrement();
@@ -120,8 +121,9 @@ public class SnapshotManager implements
// Find the source root directory path where the snapshot is taken.
final INodeDirectorySnapshottable srcRoot
= INodeDirectorySnapshottable.valueOf(fsdir.getINode(path), path);
- srcRoot.addSnapshot(snapshotID, snapshotName);
-
+ final Snapshot s = srcRoot.addSnapshot(snapshotID, snapshotName);
+ new SnapshotCreation().processRecursively(srcRoot, s.getRoot());
+
//create success, update id
snapshotID++;
numSnapshots.getAndIncrement();
@@ -152,6 +154,85 @@ public class SnapshotManager implements
srcRoot.renameSnapshot(path, oldSnapshotName, newSnapshotName);
}
+ /**
+ * Create a snapshot of subtrees by recursively coping the directory
+ * structure from the source directory to the snapshot destination directory.
+ * This creation algorithm requires O(N) running time and O(N) memory,
+ * where N = # files + # directories + # symlinks.
+ */
+ class SnapshotCreation {
+ /** Process snapshot creation recursively. */
+ private void processRecursively(final INodeDirectory srcDir,
+ final INodeDirectory dstDir) throws IOException {
+ final ReadOnlyList<INode> children = srcDir.getChildrenList(null);
+ if (!children.isEmpty()) {
+ final List<INode> inodes = new ArrayList<INode>(children.size());
+ for(final INode c : new ArrayList<INode>(ReadOnlyList.Util.asList(children))) {
+ final INode i;
+ if (c == null) {
+ i = null;
+ } else if (c instanceof INodeDirectory) {
+ //also handle INodeDirectoryWithQuota
+ i = processINodeDirectory((INodeDirectory)c);
+ } else if (c instanceof INodeFileUnderConstruction) {
+ //TODO: support INodeFileUnderConstruction
+ throw new IOException("Not yet supported.");
+ } else if (c instanceof INodeFile) {
+ i = processINodeFile(srcDir, (INodeFile)c);
+ } else if (c instanceof INodeSymlink) {
+ i = new INodeSymlink((INodeSymlink)c);
+ } else {
+ throw new AssertionError("Unknow INode type: " + c.getClass()
+ + ", inode = " + c);
+ }
+ i.setParent(dstDir);
+ inodes.add(i);
+ }
+ dstDir.setChildren(inodes);
+ }
+ }
+
+ /**
+ * Create destination INodeDirectory and make the recursive call.
+ * @return destination INodeDirectory.
+ */
+ private INodeDirectory processINodeDirectory(final INodeDirectory srcChild
+ ) throws IOException {
+ final INodeDirectory dstChild = new INodeDirectory(srcChild);
+ dstChild.setChildren(null);
+ processRecursively(srcChild, dstChild);
+ return dstChild;
+ }
+
+ /**
+ * Create destination INodeFileSnapshot and update source INode type.
+ * @return destination INodeFileSnapshot.
+ */
+ private INodeFileSnapshot processINodeFile(final INodeDirectory parent,
+ final INodeFile file) {
+ final INodeFileSnapshot snapshot = new INodeFileSnapshot(
+ file, file.computeFileSize(true));
+
+ final INodeFileWithLink srcWithLink;
+ //check source INode type
+ if (file instanceof INodeFileWithLink) {
+ srcWithLink = (INodeFileWithLink)file;
+ } else {
+ //source is an INodeFile, replace the source.
+ srcWithLink = new INodeFileWithLink(file);
+ file.removeNode();
+ parent.addChild(srcWithLink, false);
+
+ //update block map
+ namesystem.getBlockManager().addBlockCollection(srcWithLink);
+ }
+
+ //insert the snapshot to src's linked list.
+ srcWithLink.insert(snapshot);
+ return snapshot;
+ }
+ }
+
@Override
public long getNumSnapshottableDirs() {
return numSnapshottableDirs.get();
@@ -162,4 +243,4 @@ public class SnapshotManager implements
return numSnapshots.get();
}
-}
+}
\ No newline at end of file
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1415804-1416602
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1415804-1416602