You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2014/07/12 04:24:55 UTC
svn commit: r1609878 [5/9] - in
/hadoop/common/branches/YARN-1051/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs...
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat Jul 12 02:24:40 2014
@@ -85,7 +85,18 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.util.Time.now;
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
@@ -103,6 +114,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -169,7 +181,6 @@ import org.apache.hadoop.hdfs.protocol.R
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
@@ -206,8 +217,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -514,6 +523,62 @@ public class FSNamesystem implements Nam
private final NNConf nnConf;
+ private volatile boolean imageLoaded = false;
+ private final Condition cond;
+
+ private final FSImage fsImage;
+
+ /**
+ * Notify that loading of this FSDirectory is complete, and
+ * it is imageLoaded for use
+ */
+ void imageLoadComplete() {
+ Preconditions.checkState(!imageLoaded, "FSDirectory already loaded");
+ setImageLoaded();
+ }
+
+ void setImageLoaded() {
+ if(imageLoaded) return;
+ writeLock();
+ try {
+ setImageLoaded(true);
+ dir.markNameCacheInitialized();
+ cond.signalAll();
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ //This is for testing purposes only
+ @VisibleForTesting
+ boolean isImageLoaded() {
+ return imageLoaded;
+ }
+
+ // exposed for unit tests
+ protected void setImageLoaded(boolean flag) {
+ imageLoaded = flag;
+ }
+
+ /**
+ * Block until the object is imageLoaded to be used.
+ */
+ void waitForLoadingFSImage() {
+ if (!imageLoaded) {
+ writeLock();
+ try {
+ while (!imageLoaded) {
+ try {
+ cond.await(5000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignored) {
+ }
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+ }
+
/**
* Set the last allocated inode id when fsimage or editlog is loaded.
*/
@@ -555,6 +620,7 @@ public class FSNamesystem implements Nam
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
+ setImageLoaded(false);
}
@VisibleForTesting
@@ -682,6 +748,8 @@ public class FSNamesystem implements Nam
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
LOG.info("fsLock is fair:" + fair);
fsLock = new FSNamesystemLock(fair);
+ cond = fsLock.writeLock().newCondition();
+ this.fsImage = fsImage;
try {
resourceRecheckInterval = conf.getLong(
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
@@ -771,7 +839,7 @@ public class FSNamesystem implements Nam
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
this.dtSecretManager = createDelegationTokenSecretManager(conf);
- this.dir = new FSDirectory(fsImage, this, conf);
+ this.dir = new FSDirectory(this, conf);
this.snapshotManager = new SnapshotManager(dir);
this.cacheManager = new CacheManager(this, conf, blockManager);
this.safeMode = new SafeModeInfo(conf);
@@ -921,7 +989,7 @@ public class FSNamesystem implements Nam
}
writeUnlock();
}
- dir.imageLoadComplete();
+ imageLoadComplete();
}
private void startSecretManager() {
@@ -999,7 +1067,7 @@ public class FSNamesystem implements Nam
LOG.info("Starting services required for active state");
writeLock();
try {
- FSEditLog editLog = dir.fsImage.getEditLog();
+ FSEditLog editLog = getFSImage().getEditLog();
if (!editLog.isOpenForWrite()) {
// During startup, we're already open for write during initialization.
@@ -1028,12 +1096,12 @@ public class FSNamesystem implements Nam
metaSaveAsString());
}
- long nextTxId = dir.fsImage.getLastAppliedTxId() + 1;
+ long nextTxId = getFSImage().getLastAppliedTxId() + 1;
LOG.info("Will take over writing edit logs at txnid " +
nextTxId);
editLog.setNextTxId(nextTxId);
- dir.fsImage.editLog.openForWrite();
+ getFSImage().editLog.openForWrite();
}
// Enable quota checks.
@@ -1108,13 +1176,13 @@ public class FSNamesystem implements Nam
((NameNodeEditLogRoller)nnEditLogRoller.getRunnable()).stop();
nnEditLogRoller.interrupt();
}
- if (dir != null && dir.fsImage != null) {
- if (dir.fsImage.editLog != null) {
- dir.fsImage.editLog.close();
+ if (dir != null && getFSImage() != null) {
+ if (getFSImage().editLog != null) {
+ getFSImage().editLog.close();
}
// Update the fsimage with the last txid that we wrote
// so that the tailer starts from the right spot.
- dir.fsImage.updateLastAppliedTxIdFromWritten();
+ getFSImage().updateLastAppliedTxIdFromWritten();
}
if (cacheManager != null) {
cacheManager.stopMonitorThread();
@@ -1137,9 +1205,9 @@ public class FSNamesystem implements Nam
*/
void startStandbyServices(final Configuration conf) throws IOException {
LOG.info("Starting services required for standby state");
- if (!dir.fsImage.editLog.isOpenForRead()) {
+ if (!getFSImage().editLog.isOpenForRead()) {
// During startup, we're already open for read.
- dir.fsImage.editLog.initSharedJournalsForRead();
+ getFSImage().editLog.initSharedJournalsForRead();
}
blockManager.setPostponeBlocksFromFuture(true);
@@ -1186,8 +1254,8 @@ public class FSNamesystem implements Nam
if (editLogTailer != null) {
editLogTailer.stop();
}
- if (dir != null && dir.fsImage != null && dir.fsImage.editLog != null) {
- dir.fsImage.editLog.close();
+ if (dir != null && getFSImage() != null && getFSImage().editLog != null) {
+ getFSImage().editLog.close();
}
}
@@ -1430,9 +1498,9 @@ public class FSNamesystem implements Nam
* Version of @see #getNamespaceInfo() that is not protected by a lock.
*/
NamespaceInfo unprotectedGetNamespaceInfo() {
- return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(),
+ return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(),
getClusterId(), getBlockPoolId(),
- dir.fsImage.getStorage().getCTime());
+ getFSImage().getStorage().getCTime());
}
/**
@@ -1450,12 +1518,10 @@ public class FSNamesystem implements Nam
try {
stopActiveServices();
stopStandbyServices();
- if (dir != null) {
- dir.close();
- }
} catch (IOException ie) {
- LOG.error("Error closing FSDirectory", ie);
+ } finally {
IOUtils.cleanup(LOG, dir);
+ IOUtils.cleanup(LOG, fsImage);
}
}
}
@@ -1840,6 +1906,7 @@ public class FSNamesystem implements Nam
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2115,6 +2182,7 @@ public class FSNamesystem implements Nam
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2242,6 +2310,8 @@ public class FSNamesystem implements Nam
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2326,9 +2396,6 @@ public class FSNamesystem implements Nam
}
checkFsObjectLimit();
- final DatanodeDescriptor clientNode =
- blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
-
INodeFile newNode = null;
// Always do an implicit mkdirs for parent directory tree.
@@ -2336,7 +2403,7 @@ public class FSNamesystem implements Nam
if (parent != null && mkdirsRecursively(parent.toString(),
permissions, true, now())) {
newNode = dir.addFile(src, permissions, replication, blockSize,
- holder, clientMachine, clientNode);
+ holder, clientMachine);
}
if (newNode == null) {
@@ -2411,10 +2478,8 @@ public class FSNamesystem implements Nam
throw new IOException("append: lastBlock=" + lastBlock +
" of src=" + src + " is not sufficiently replicated yet.");
}
- final DatanodeDescriptor clientNode =
- blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
- return prepareFileForWrite(src, myFile, holder, clientMachine, clientNode,
- true, iip.getLatestSnapshotId(), logRetryCache);
+ return prepareFileForWrite(src, myFile, holder, clientMachine, true,
+ iip.getLatestSnapshotId(), logRetryCache);
} catch (IOException ie) {
NameNode.stateChangeLog.warn("DIR* NameSystem.append: " +ie.getMessage());
throw ie;
@@ -2429,7 +2494,6 @@ public class FSNamesystem implements Nam
* @param file existing file object
* @param leaseHolder identifier of the lease holder on this file
* @param clientMachine identifier of the client machine
- * @param clientNode if the client is collocated with a DN, that DN's descriptor
* @param writeToEditLog whether to persist this change to the edit log
* @param logRetryCache whether to record RPC ids in editlog for retry cache
* rebuilding
@@ -2438,12 +2502,12 @@ public class FSNamesystem implements Nam
* @throws IOException
*/
LocatedBlock prepareFileForWrite(String src, INodeFile file,
- String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
- boolean writeToEditLog, int latestSnapshot, boolean logRetryCache)
+ String leaseHolder, String clientMachine,
+ boolean writeToEditLog,
+ int latestSnapshot, boolean logRetryCache)
throws IOException {
file = file.recordModification(latestSnapshot);
- final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine,
- clientNode);
+ final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine);
leaseManager.addLease(cons.getFileUnderConstructionFeature()
.getClientName(), src);
@@ -2452,7 +2516,7 @@ public class FSNamesystem implements Nam
if (ret != null) {
// update the quota: use the preferred block size for UC block
final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
- dir.updateSpaceConsumed(src, 0, diff);
+ dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication());
}
if (writeToEditLog) {
@@ -2524,10 +2588,10 @@ public class FSNamesystem implements Nam
// We found the lease for this file. And surprisingly the original
// holder is trying to recreate this file. This should never occur.
//
+
if (!force && lease != null) {
Lease leaseFile = leaseManager.getLeaseByPath(src);
- if ((leaseFile != null && leaseFile.equals(lease)) ||
- lease.getHolder().equals(holder)) {
+ if (leaseFile != null && leaseFile.equals(lease)) {
throw new AlreadyBeingCreatedException(
"failed to create file " + src + " for " + holder +
" for client " + clientMachine +
@@ -2700,9 +2764,10 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INodeFile pendingFile = analyzeFileState(
+ FileState fileState = analyzeFileState(
src, fileId, clientName, previous, onRetryBlock);
- src = pendingFile.getFullPathName();
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
// This is a retry. Just return the last block if having locations.
@@ -2715,7 +2780,8 @@ public class FSNamesystem implements Nam
+ maxBlocksPerFile);
}
blockSize = pendingFile.getPreferredBlockSize();
- clientNode = pendingFile.getFileUnderConstructionFeature().getClientNode();
+ clientNode = blockManager.getDatanodeManager().getDatanodeByHost(
+ pendingFile.getFileUnderConstructionFeature().getClientMachine());
replication = pendingFile.getFileReplication();
} finally {
readUnlock();
@@ -2730,14 +2796,17 @@ public class FSNamesystem implements Nam
Block newBlock = null;
long offset;
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
// Run the full analysis again, since things could have changed
// while chooseTarget() was executing.
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INodeFile pendingFile =
+ FileState fileState =
analyzeFileState(src, fileId, clientName, previous, onRetryBlock);
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null) {
if (onRetryBlock[0].getLocations().length > 0) {
@@ -2773,7 +2842,17 @@ public class FSNamesystem implements Nam
return makeLocatedBlock(newBlock, targets, offset);
}
- INodeFile analyzeFileState(String src,
+ static class FileState {
+ public final INodeFile inode;
+ public final String path;
+
+ public FileState(INodeFile inode, String fullPath) {
+ this.inode = inode;
+ this.path = fullPath;
+ }
+ }
+
+ FileState analyzeFileState(String src,
long fileId,
String clientName,
ExtendedBlock previous,
@@ -2861,7 +2940,7 @@ public class FSNamesystem implements Nam
onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
offset);
- return pendingFile;
+ return new FileState(pendingFile, src);
} else {
// Case 3
throw new IOException("Cannot allocate block in " + src + ": " +
@@ -2874,7 +2953,7 @@ public class FSNamesystem implements Nam
if (!checkFileProgress(pendingFile, false)) {
throw new NotReplicatedYetException("Not replicated yet: " + src);
}
- return pendingFile;
+ return new FileState(pendingFile, src);
}
LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
@@ -2920,7 +2999,9 @@ public class FSNamesystem implements Nam
if (inode != null) src = inode.getFullPathName();
}
final INodeFile file = checkLease(src, clientName, inode, fileId);
- clientnode = file.getFileUnderConstructionFeature().getClientNode();
+ String clientMachine = file.getFileUnderConstructionFeature()
+ .getClientMachine();
+ clientnode = blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
preferredblocksize = file.getPreferredBlockSize();
//find datanode storages
@@ -2952,6 +3033,7 @@ public class FSNamesystem implements Nam
}
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3019,10 +3101,9 @@ public class FSNamesystem implements Nam
: "Holder " + holder + " does not have any open files."));
}
// No further modification is allowed on a deleted file.
- // A file is considered deleted, if it has no parent or is marked
+ // A file is considered deleted, if it is not in the inodeMap or is marked
// as deleted in the snapshot feature.
- if (file.getParent() == null || (file.isWithSnapshot() &&
- file.getFileWithSnapshotFeature().isCurrentFileDeleted())) {
+ if (isFileDeleted(file)) {
throw new FileNotFoundException(src);
}
String clientName = file.getFileUnderConstructionFeature().getClientName();
@@ -3050,6 +3131,7 @@ public class FSNamesystem implements Nam
boolean success = false;
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3249,6 +3331,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot rename " + src);
+ waitForLoadingFSImage();
src = FSDirectory.resolvePath(src, srcComponents, dir);
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
checkOperation(OperationCategory.WRITE);
@@ -3356,6 +3439,7 @@ public class FSNamesystem implements Nam
false);
}
+ waitForLoadingFSImage();
long mtime = now();
dir.renameTo(src, dst, mtime, options);
getEditLog().logRename(src, dst, mtime, logRetryCache, options);
@@ -3429,6 +3513,8 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean ret = false;
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3451,7 +3537,7 @@ public class FSNamesystem implements Nam
getEditLog().logDelete(src, mtime, logRetryCache);
incrDeletedFileCount(filesRemoved);
// Blocks/INodes will be handled later
- removePathAndBlocks(src, null, null);
+ removePathAndBlocks(src, null, removedINodes, true);
ret = true;
} finally {
writeUnlock();
@@ -3460,13 +3546,6 @@ public class FSNamesystem implements Nam
removeBlocks(collectedBlocks); // Incremental deletion of blocks
collectedBlocks.clear();
- dir.writeLock();
- try {
- dir.removeFromInodeMap(removedINodes);
- } finally {
- dir.writeUnlock();
- }
- removedINodes.clear();
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "
+ src +" is removed");
@@ -3504,14 +3583,24 @@ public class FSNamesystem implements Nam
* @param blocks Containing the list of blocks to be deleted from blocksMap
* @param removedINodes Containing the list of inodes to be removed from
* inodesMap
+ * @param acquireINodeMapLock Whether to acquire the lock for inode removal
*/
void removePathAndBlocks(String src, BlocksMapUpdateInfo blocks,
- List<INode> removedINodes) {
+ List<INode> removedINodes, final boolean acquireINodeMapLock) {
assert hasWriteLock();
leaseManager.removeLeaseWithPrefixPath(src);
// remove inodes from inodesMap
if (removedINodes != null) {
- dir.removeFromInodeMap(removedINodes);
+ if (acquireINodeMapLock) {
+ dir.writeLock();
+ }
+ try {
+ dir.removeFromInodeMap(removedINodes);
+ } finally {
+ if (acquireINodeMapLock) {
+ dir.writeUnlock();
+ }
+ }
removedINodes.clear();
}
if (blocks == null) {
@@ -3673,7 +3762,7 @@ public class FSNamesystem implements Nam
src = FSDirectory.resolvePath(src, pathComponents, dir);
status = mkdirsInternal(pc, src, permissions, createParent);
if (status) {
- resultingStat = dir.getFileInfo(src, false);
+ resultingStat = getAuditFileInfo(src, false);
}
} finally {
writeUnlock();
@@ -3902,6 +3991,8 @@ public class FSNamesystem implements Nam
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -4103,6 +4194,7 @@ public class FSNamesystem implements Nam
INodeFile pendingFile, int latestSnapshot) throws IOException,
UnresolvedLinkException {
assert hasWriteLock();
+
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
Preconditions.checkArgument(uc != null);
leaseManager.removeLease(uc.getClientName(), src);
@@ -4114,6 +4206,7 @@ public class FSNamesystem implements Nam
// since we just remove the uc feature from pendingFile
final INodeFile newFile = pendingFile.toCompleteFile(now());
+ waitForLoadingFSImage();
// close file and persist block allocations for this file
closeFile(src, newFile);
@@ -4172,6 +4265,7 @@ public class FSNamesystem implements Nam
+ ")");
checkOperation(OperationCategory.WRITE);
String src = "";
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -4431,7 +4525,7 @@ public class FSNamesystem implements Nam
* @return registration ID
*/
String getRegistrationID() {
- return Storage.getRegistrationID(dir.fsImage.getStorage());
+ return Storage.getRegistrationID(getFSImage().getStorage());
}
/**
@@ -4517,7 +4611,7 @@ public class FSNamesystem implements Nam
*/
private void closeFile(String path, INodeFile file) {
assert hasWriteLock();
- dir.waitForReady();
+ waitForLoadingFSImage();
// file is closed
getEditLog().logCloseFile(path, file);
if (NameNode.stateChangeLog.isDebugEnabled()) {
@@ -4541,7 +4635,7 @@ public class FSNamesystem implements Nam
boolean createParent, boolean logRetryCache)
throws UnresolvedLinkException, FileAlreadyExistsException,
QuotaExceededException, SnapshotAccessControlException, AclException {
- dir.waitForReady();
+ waitForLoadingFSImage();
final long modTime = now();
if (createParent) {
@@ -4647,7 +4741,7 @@ public class FSNamesystem implements Nam
}
public FSImage getFSImage() {
- return dir.fsImage;
+ return fsImage;
}
public FSEditLog getEditLog() {
@@ -5804,7 +5898,7 @@ public class FSNamesystem implements Nam
boolean ignoreEmptyDir, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException {
if (!pc.isSuperUser()) {
- dir.waitForReady();
+ waitForLoadingFSImage();
readLock();
try {
pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
@@ -6156,6 +6250,16 @@ public class FSNamesystem implements Nam
return blockId;
}
+ private boolean isFileDeleted(INodeFile file) {
+ // Not in the inodeMap or in the snapshot but marked deleted.
+ if (dir.getInode(file.getId()) == null ||
+ file.getParent() == null || (file.isWithSnapshot() &&
+ file.getFileWithSnapshotFeature().isCurrentFileDeleted())) {
+ return true;
+ }
+ return false;
+ }
+
private INodeFile checkUCBlock(ExtendedBlock block,
String clientName) throws IOException {
assert hasWriteLock();
@@ -6172,7 +6276,7 @@ public class FSNamesystem implements Nam
// check file inode
final INodeFile file = ((INode)storedBlock.getBlockCollection()).asFile();
- if (file == null || !file.isUnderConstruction()) {
+ if (file == null || !file.isUnderConstruction() || isFileDeleted(file)) {
throw new IOException("The file " + storedBlock +
" belonged to does not exist or it is not under construction.");
}
@@ -6271,6 +6375,7 @@ public class FSNamesystem implements Nam
+ ", newNodes=" + Arrays.asList(newNodes)
+ ", clientName=" + clientName
+ ")");
+ waitForLoadingFSImage();
writeLock();
boolean success = false;
try {
@@ -6971,7 +7076,7 @@ public class FSNamesystem implements Nam
@Override // NameNodeMXBean
public String getClusterId() {
- return dir.fsImage.getStorage().getClusterID();
+ return getFSImage().getStorage().getClusterID();
}
@Override // NameNodeMXBean
@@ -7412,7 +7517,7 @@ public class FSNamesystem implements Nam
*/
SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException {
- SnapshotDiffInfo diffs = null;
+ SnapshotDiffReport diffs;
checkOperation(OperationCategory.READ);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
@@ -7426,13 +7531,11 @@ public class FSNamesystem implements Nam
} finally {
readUnlock();
}
-
+
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
}
- return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
- path, fromSnapshot, toSnapshot,
- Collections.<DiffReportEntry> emptyList());
+ return diffs;
}
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
@@ -7501,7 +7604,7 @@ public class FSNamesystem implements Nam
* Remove a list of INodeDirectorySnapshottable from the SnapshotManager
* @param toRemove the list of INodeDirectorySnapshottable to be removed
*/
- void removeSnapshottableDirs(List<INodeDirectorySnapshottable> toRemove) {
+ void removeSnapshottableDirs(List<INodeDirectory> toRemove) {
if (snapshotManager != null) {
snapshotManager.removeSnapshottable(toRemove);
}
@@ -7651,6 +7754,7 @@ public class FSNamesystem implements Nam
// roll the edit log to make sure the standby NameNode can tail
getFSImage().rollEditLog();
}
+ getFSImage().updateStorageVersion();
getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK,
NameNodeFile.IMAGE);
} finally {
@@ -8117,12 +8221,11 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set XAttr on " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
- if (isPermissionEnabled) {
- checkOwner(pc, src);
- checkPathAccess(pc, src, FsAction.WRITE);
- }
- dir.setXAttr(src, xAttr, flag);
- getEditLog().logSetXAttr(src, xAttr, logRetryCache);
+ checkXAttrChangeAccess(src, xAttr, pc);
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ dir.setXAttrs(src, xAttrs, flag);
+ getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -8238,14 +8341,13 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot remove XAttr entry on " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
- if (isPermissionEnabled) {
- checkOwner(pc, src);
- checkPathAccess(pc, src, FsAction.WRITE);
- }
-
- XAttr removedXAttr = dir.removeXAttr(src, xAttr);
- if (removedXAttr != null) {
- getEditLog().logRemoveXAttr(src, removedXAttr);
+ checkXAttrChangeAccess(src, xAttr, pc);
+
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ List<XAttr> removedXAttrs = dir.removeXAttrs(src, xAttrs);
+ if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
+ getEditLog().logRemoveXAttrs(src, removedXAttrs);
}
resultingStat = getAuditFileInfo(src, false);
} catch (AccessControlException e) {
@@ -8258,6 +8360,21 @@ public class FSNamesystem implements Nam
logAuditEvent(true, "removeXAttr", src, null, resultingStat);
}
+ private void checkXAttrChangeAccess(String src, XAttr xAttr,
+ FSPermissionChecker pc) throws UnresolvedLinkException,
+ AccessControlException {
+ if (isPermissionEnabled && xAttr.getNameSpace() == XAttr.NameSpace.USER) {
+ final INode inode = dir.getINode(src);
+ if (inode.isDirectory() && inode.getFsPermission().getStickyBit()) {
+ if (!pc.isSuperUser()) {
+ checkOwner(pc, src);
+ }
+ } else {
+ checkPathAccess(pc, src, FsAction.WRITE);
+ }
+ }
+ }
+
/**
* Default AuditLogger implementation; used when no access logger is
* defined in the config file. It can also be explicitly listed in the
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java Sat Jul 12 02:24:40 2014
@@ -32,15 +32,10 @@ import org.apache.hadoop.hdfs.server.nam
public class FileUnderConstructionFeature implements INode.Feature {
private String clientName; // lease holder
private final String clientMachine;
- // if client is a cluster node too.
- private final DatanodeDescriptor clientNode;
- public FileUnderConstructionFeature(final String clientName,
- final String clientMachine,
- final DatanodeDescriptor clientNode) {
+ public FileUnderConstructionFeature(final String clientName, final String clientMachine) {
this.clientName = clientName;
this.clientMachine = clientMachine;
- this.clientNode = clientNode;
}
public String getClientName() {
@@ -55,10 +50,6 @@ public class FileUnderConstructionFeatur
return clientMachine;
}
- public DatanodeDescriptor getClientNode() {
- return clientNode;
- }
-
/**
* Update the length for the last block
*
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java Sat Jul 12 02:24:40 2014
@@ -97,14 +97,12 @@ public interface INodeAttributes {
@Override
public final String getUserName() {
- final int n = (int)PermissionStatusFormat.USER.retrieve(permission);
- return SerialNumberManager.INSTANCE.getUser(n);
+ return PermissionStatusFormat.getUser(permission);
}
@Override
public final String getGroupName() {
- final int n = (int)PermissionStatusFormat.GROUP.retrieve(permission);
- return SerialNumberManager.INSTANCE.getGroup(n);
+ return PermissionStatusFormat.getGroup(permission);
}
@Override
@@ -114,7 +112,7 @@ public interface INodeAttributes {
@Override
public final short getFsPermissionShort() {
- return (short)PermissionStatusFormat.MODE.retrieve(permission);
+ return PermissionStatusFormat.getMode(permission);
}
@Override
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Sat Jul 12 02:24:40 2014
@@ -26,15 +26,14 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
-import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.Diff.ListType;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -104,11 +103,6 @@ public class INodeDirectory extends INod
return this;
}
- /** Is this a snapshottable directory? */
- public boolean isSnapshottable() {
- return false;
- }
-
void setQuota(long nsQuota, long dsQuota) {
DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
if (quota != null) {
@@ -188,7 +182,7 @@ public class INodeDirectory extends INod
public final boolean isWithSnapshot() {
return getDirectoryWithSnapshotFeature() != null;
}
-
+
public DirectoryDiffList getDiffs() {
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
return sf != null ? sf.getDiffs() : null;
@@ -206,50 +200,71 @@ public class INodeDirectory extends INod
return super.toDetailString() + (sf == null ? "" : ", " + sf.getDiffs());
}
- /** Replace itself with an {@link INodeDirectorySnapshottable}. */
- public INodeDirectorySnapshottable replaceSelf4INodeDirectorySnapshottable(
- int latestSnapshotId, final INodeMap inodeMap)
- throws QuotaExceededException {
- Preconditions.checkState(!(this instanceof INodeDirectorySnapshottable),
- "this is already an INodeDirectorySnapshottable, this=%s", this);
- final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(this);
- replaceSelf(s, inodeMap).getDirectoryWithSnapshotFeature().getDiffs()
- .saveSelf2Snapshot(latestSnapshotId, s, this);
- return s;
- }
-
- /** Replace itself with {@link INodeDirectory}. */
- public INodeDirectory replaceSelf4INodeDirectory(final INodeMap inodeMap) {
- Preconditions.checkState(getClass() != INodeDirectory.class,
- "the class is already INodeDirectory, this=%s", this);
- return replaceSelf(new INodeDirectory(this, true, this.getFeatures()),
- inodeMap);
+ public DirectorySnapshottableFeature getDirectorySnapshottableFeature() {
+ return getFeature(DirectorySnapshottableFeature.class);
}
- /** Replace itself with the given directory. */
- private final <N extends INodeDirectory> N replaceSelf(final N newDir,
- final INodeMap inodeMap) {
- final INodeReference ref = getParentReference();
- if (ref != null) {
- ref.setReferredINode(newDir);
- if (inodeMap != null) {
- inodeMap.put(newDir);
- }
- } else {
- final INodeDirectory parent = getParent();
- Preconditions.checkArgument(parent != null, "parent is null, this=%s", this);
- parent.replaceChild(this, newDir, inodeMap);
+ public boolean isSnapshottable() {
+ return getDirectorySnapshottableFeature() != null;
+ }
+
+ public Snapshot getSnapshot(byte[] snapshotName) {
+ return getDirectorySnapshottableFeature().getSnapshot(snapshotName);
+ }
+
+ public void setSnapshotQuota(int snapshotQuota) {
+ getDirectorySnapshottableFeature().setSnapshotQuota(snapshotQuota);
+ }
+
+ public Snapshot addSnapshot(int id, String name) throws SnapshotException,
+ QuotaExceededException {
+ return getDirectorySnapshottableFeature().addSnapshot(this, id, name);
+ }
+
+ public Snapshot removeSnapshot(String snapshotName,
+ BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
+ throws SnapshotException {
+ return getDirectorySnapshottableFeature().removeSnapshot(this,
+ snapshotName, collectedBlocks, removedINodes);
+ }
+
+ public void renameSnapshot(String path, String oldName, String newName)
+ throws SnapshotException {
+ getDirectorySnapshottableFeature().renameSnapshot(path, oldName, newName);
+ }
+
+ /** add DirectorySnapshottableFeature */
+ public void addSnapshottableFeature() {
+ Preconditions.checkState(!isSnapshottable(),
+ "this is already snapshottable, this=%s", this);
+ DirectoryWithSnapshotFeature s = this.getDirectoryWithSnapshotFeature();
+ final DirectorySnapshottableFeature snapshottable =
+ new DirectorySnapshottableFeature(s);
+ if (s != null) {
+ this.removeFeature(s);
}
- clear();
- return newDir;
+ this.addFeature(snapshottable);
}
-
+
+ /** remove DirectorySnapshottableFeature */
+ public void removeSnapshottableFeature() {
+ DirectorySnapshottableFeature s = getDirectorySnapshottableFeature();
+ Preconditions.checkState(s != null,
+ "The dir does not have snapshottable feature: this=%s", this);
+ this.removeFeature(s);
+ if (s.getDiffs().asList().size() > 0) {
+ // add a DirectoryWithSnapshotFeature back
+ DirectoryWithSnapshotFeature sf = new DirectoryWithSnapshotFeature(
+ s.getDiffs());
+ addFeature(sf);
+ }
+ }
+
/**
* Replace the given child with a new child. Note that we no longer need to
* replace an normal INodeDirectory or INodeFile into an
* INodeDirectoryWithSnapshot or INodeFileUnderConstruction. The only cases
- * for child replacement is for {@link INodeDirectorySnapshottable} and
- * reference nodes.
+ * for child replacement is for reference nodes.
*/
public void replaceChild(INode oldChild, final INode newChild,
final INodeMap inodeMap) {
@@ -365,7 +380,7 @@ public class INodeDirectory extends INod
* children list nor in any snapshot; otherwise the snapshot id of the
* corresponding snapshot diff list.
*/
- int searchChild(INode inode) {
+ public int searchChild(INode inode) {
INode child = getChild(inode.getLocalNameBytes(), Snapshot.CURRENT_STATE_ID);
if (child != inode) {
// inode is not in parent's children list, thus inode must be in
@@ -764,7 +779,9 @@ public class INodeDirectory extends INod
public boolean metadataEquals(INodeDirectoryAttributes other) {
return other != null
&& getQuotaCounts().equals(other.getQuotaCounts())
- && getPermissionLong() == other.getPermissionLong();
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
}
/*
@@ -822,6 +839,11 @@ public class INodeDirectory extends INod
};
}
});
+
+ final DirectorySnapshottableFeature s = getDirectorySnapshottableFeature();
+ if (s != null) {
+ s.dumpTreeRecursively(this, out, prefix, snapshot);
+ }
}
/**
@@ -830,7 +852,7 @@ public class INodeDirectory extends INod
* @param subs The subtrees.
*/
@VisibleForTesting
- protected static void dumpTreeRecursively(PrintWriter out,
+ public static void dumpTreeRecursively(PrintWriter out,
StringBuilder prefix, Iterable<SnapshotAndINode> subs) {
if (subs != null) {
for(final Iterator<SnapshotAndINode> i = subs.iterator(); i.hasNext();) {
@@ -843,7 +865,7 @@ public class INodeDirectory extends INod
}
/** A pair of Snapshot and INode objects. */
- protected static class SnapshotAndINode {
+ public static class SnapshotAndINode {
public final int snapshotId;
public final INode inode;
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java Sat Jul 12 02:24:40 2014
@@ -53,8 +53,10 @@ public interface INodeDirectoryAttribute
@Override
public boolean metadataEquals(INodeDirectoryAttributes other) {
return other != null
- && this.getQuotaCounts().equals(other.getQuotaCounts())
- && getPermissionLong() == other.getPermissionLong();
+ && getQuotaCounts().equals(other.getQuotaCounts())
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Sat Jul 12 02:24:40 2014
@@ -33,13 +33,13 @@ import org.apache.hadoop.hdfs.protocol.Q
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -72,37 +72,29 @@ public class INodeFile extends INodeWith
}
/** Format: [16 bits for replication][48 bits for PreferredBlockSize] */
- static class HeaderFormat {
- /** Number of bits for Block size */
- static final int BLOCKBITS = 48;
- /** Header mask 64-bit representation */
- static final long HEADERMASK = 0xffffL << BLOCKBITS;
- static final long MAX_BLOCK_SIZE = ~HEADERMASK;
-
- static short getReplication(long header) {
- return (short) ((header & HEADERMASK) >> BLOCKBITS);
+ static enum HeaderFormat {
+ PREFERRED_BLOCK_SIZE(null, 48, 1),
+ REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 16, 1);
+
+ private final LongBitFormat BITS;
+
+ private HeaderFormat(LongBitFormat previous, int length, long min) {
+ BITS = new LongBitFormat(name(), previous, length, min);
}
- static long combineReplication(long header, short replication) {
- if (replication <= 0) {
- throw new IllegalArgumentException(
- "Unexpected value for the replication: " + replication);
- }
- return ((long)replication << BLOCKBITS) | (header & MAX_BLOCK_SIZE);
+ static short getReplication(long header) {
+ return (short)REPLICATION.BITS.retrieve(header);
}
-
+
static long getPreferredBlockSize(long header) {
- return header & MAX_BLOCK_SIZE;
+ return PREFERRED_BLOCK_SIZE.BITS.retrieve(header);
}
- static long combinePreferredBlockSize(long header, long blockSize) {
- if (blockSize < 0) {
- throw new IllegalArgumentException("Block size < 0: " + blockSize);
- } else if (blockSize > MAX_BLOCK_SIZE) {
- throw new IllegalArgumentException("Block size = " + blockSize
- + " > MAX_BLOCK_SIZE = " + MAX_BLOCK_SIZE);
- }
- return (header & HEADERMASK) | (blockSize & MAX_BLOCK_SIZE);
+ static long toLong(long preferredBlockSize, short replication) {
+ long h = 0;
+ h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
+ h = REPLICATION.BITS.combine(replication, h);
+ return h;
}
}
@@ -114,8 +106,7 @@ public class INodeFile extends INodeWith
long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize) {
super(id, name, permissions, mtime, atime);
- header = HeaderFormat.combineReplication(header, replication);
- header = HeaderFormat.combinePreferredBlockSize(header, preferredBlockSize);
+ header = HeaderFormat.toLong(preferredBlockSize, replication);
this.blocks = blklist;
}
@@ -144,6 +135,15 @@ public class INodeFile extends INodeWith
return this;
}
+ @Override
+ public boolean metadataEquals(INodeFileAttributes other) {
+ return other != null
+ && getHeaderLong()== other.getHeaderLong()
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
+ }
+
/* Start of Under-Construction Feature */
/**
@@ -161,12 +161,11 @@ public class INodeFile extends INodeWith
}
/** Convert this file to an {@link INodeFileUnderConstruction}. */
- INodeFile toUnderConstruction(String clientName, String clientMachine,
- DatanodeDescriptor clientNode) {
+ INodeFile toUnderConstruction(String clientName, String clientMachine) {
Preconditions.checkState(!isUnderConstruction(),
"file is already under construction");
FileUnderConstructionFeature uc = new FileUnderConstructionFeature(
- clientName, clientMachine, clientNode);
+ clientName, clientMachine);
addFeature(uc);
return this;
}
@@ -340,7 +339,7 @@ public class INodeFile extends INodeWith
/** Set the replication factor of this file. */
public final void setFileReplication(short replication) {
- header = HeaderFormat.combineReplication(header, replication);
+ header = HeaderFormat.REPLICATION.BITS.combine(replication, header);
}
/** Set the replication factor of this file. */
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java Sat Jul 12 02:24:40 2014
@@ -36,6 +36,8 @@ public interface INodeFileAttributes ext
/** @return the header as a long. */
public long getHeaderLong();
+ public boolean metadataEquals(INodeFileAttributes other);
+
/** A copy of the inode file attributes */
public static class SnapshotCopy extends INodeAttributes.SnapshotCopy
implements INodeFileAttributes {
@@ -46,9 +48,7 @@ public interface INodeFileAttributes ext
short replication, long preferredBlockSize, XAttrFeature xAttrsFeature) {
super(name, permissions, aclFeature, modificationTime, accessTime,
xAttrsFeature);
-
- final long h = HeaderFormat.combineReplication(0L, replication);
- header = HeaderFormat.combinePreferredBlockSize(h, preferredBlockSize);
+ header = HeaderFormat.toLong(preferredBlockSize, replication);
}
public SnapshotCopy(INodeFile file) {
@@ -70,5 +70,14 @@ public interface INodeFileAttributes ext
public long getHeaderLong() {
return header;
}
+
+ @Override
+ public boolean metadataEquals(INodeFileAttributes other) {
+ return other != null
+ && getHeaderLong()== other.getHeaderLong()
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
+ }
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java Sat Jul 12 02:24:40 2014
@@ -435,21 +435,28 @@ public abstract class INodeReference ext
}
}
+ /**
+ * @return the WithName/DstReference node contained in the given snapshot.
+ */
public INodeReference getParentRef(int snapshotId) {
- // when the given snapshotId is CURRENT_STATE_ID, it is possible that we
- // do not know where the corresponding inode belongs, thus we simply
- // return the last reference node
- if (snapshotId == Snapshot.CURRENT_STATE_ID) {
- return this.getParentReference() != null ? this.getParentReference()
- : this.getLastWithName();
- }
- // otherwise we search the withNameList
- for (int i = 0; i < withNameList.size(); i++) {
- if (snapshotId <= withNameList.get(i).lastSnapshotId) {
- return withNameList.get(i);
+ int start = 0;
+ int end = withNameList.size() - 1;
+ while (start < end) {
+ int mid = start + (end - start) / 2;
+ int sid = withNameList.get(mid).lastSnapshotId;
+ if (sid == snapshotId) {
+ return withNameList.get(mid);
+ } else if (sid < snapshotId) {
+ start = mid + 1;
+ } else {
+ end = mid;
}
}
- return this.getParentReference();
+ if (withNameList.get(start).lastSnapshotId >= snapshotId) {
+ return withNameList.get(start);
+ } else {
+ return this.getParentReference();
+ }
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java Sat Jul 12 02:24:40 2014
@@ -21,9 +21,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.namenode.INode.Feature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
-import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
import com.google.common.base.Preconditions;
@@ -36,26 +35,28 @@ import com.google.common.base.Preconditi
public abstract class INodeWithAdditionalFields extends INode
implements LinkedElement {
static enum PermissionStatusFormat {
- MODE(0, 16),
- GROUP(MODE.OFFSET + MODE.LENGTH, 25),
- USER(GROUP.OFFSET + GROUP.LENGTH, 23);
+ MODE(null, 16),
+ GROUP(MODE.BITS, 25),
+ USER(GROUP.BITS, 23);
- final int OFFSET;
- final int LENGTH; //bit length
- final long MASK;
+ final LongBitFormat BITS;
- PermissionStatusFormat(int offset, int length) {
- OFFSET = offset;
- LENGTH = length;
- MASK = ((-1L) >>> (64 - LENGTH)) << OFFSET;
+ private PermissionStatusFormat(LongBitFormat previous, int length) {
+ BITS = new LongBitFormat(name(), previous, length, 0);
}
- long retrieve(long record) {
- return (record & MASK) >>> OFFSET;
+ static String getUser(long permission) {
+ final int n = (int)USER.BITS.retrieve(permission);
+ return SerialNumberManager.INSTANCE.getUser(n);
}
- long combine(long bits, long record) {
- return (record & ~MASK) | (bits << OFFSET);
+ static String getGroup(long permission) {
+ final int n = (int)GROUP.BITS.retrieve(permission);
+ return SerialNumberManager.INSTANCE.getGroup(n);
+ }
+
+ static short getMode(long permission) {
+ return (short)MODE.BITS.retrieve(permission);
}
/** Encode the {@link PermissionStatus} to a long. */
@@ -63,12 +64,12 @@ public abstract class INodeWithAdditiona
long permission = 0L;
final int user = SerialNumberManager.INSTANCE.getUserSerialNumber(
ps.getUserName());
- permission = USER.combine(user, permission);
+ permission = USER.BITS.combine(user, permission);
final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber(
ps.getGroupName());
- permission = GROUP.combine(group, permission);
+ permission = GROUP.BITS.combine(group, permission);
final int mode = ps.getPermission().toShort();
- permission = MODE.combine(mode, permission);
+ permission = MODE.BITS.combine(mode, permission);
return permission;
}
}
@@ -162,7 +163,7 @@ public abstract class INodeWithAdditiona
}
private final void updatePermissionStatus(PermissionStatusFormat f, long n) {
- this.permission = f.combine(n, permission);
+ this.permission = f.BITS.combine(n, permission);
}
@Override
@@ -170,9 +171,7 @@ public abstract class INodeWithAdditiona
if (snapshotId != Snapshot.CURRENT_STATE_ID) {
return getSnapshotINode(snapshotId).getUserName();
}
-
- int n = (int)PermissionStatusFormat.USER.retrieve(permission);
- return SerialNumberManager.INSTANCE.getUser(n);
+ return PermissionStatusFormat.getUser(permission);
}
@Override
@@ -186,9 +185,7 @@ public abstract class INodeWithAdditiona
if (snapshotId != Snapshot.CURRENT_STATE_ID) {
return getSnapshotINode(snapshotId).getGroupName();
}
-
- int n = (int)PermissionStatusFormat.GROUP.retrieve(permission);
- return SerialNumberManager.INSTANCE.getGroup(n);
+ return PermissionStatusFormat.getGroup(permission);
}
@Override
@@ -208,7 +205,7 @@ public abstract class INodeWithAdditiona
@Override
public final short getFsPermissionShort() {
- return (short)PermissionStatusFormat.MODE.retrieve(permission);
+ return PermissionStatusFormat.getMode(permission);
}
@Override
void setPermission(FsPermission permission) {
@@ -318,8 +315,9 @@ public abstract class INodeWithAdditiona
}
protected <T extends Feature> T getFeature(Class<? extends Feature> clazz) {
+ Preconditions.checkArgument(clazz != null);
for (Feature f : features) {
- if (f.getClass() == clazz) {
+ if (clazz.isAssignableFrom(f.getClass())) {
@SuppressWarnings("unchecked")
T ret = (T) f;
return ret;
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java Sat Jul 12 02:24:40 2014
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import com.google.common.base.Preconditions;
@@ -208,8 +207,7 @@ public class INodesInPath {
final byte[] childName = components[count + 1];
// check if the next byte[] in components is for ".snapshot"
- if (isDotSnapshotDir(childName)
- && isDir && dir instanceof INodeDirectorySnapshottable) {
+ if (isDotSnapshotDir(childName) && isDir && dir.isSnapshottable()) {
// skip the ".snapshot" in components
count++;
index++;
@@ -222,8 +220,7 @@ public class INodesInPath {
break;
}
// Resolve snapshot root
- final Snapshot s = ((INodeDirectorySnapshottable)dir).getSnapshot(
- components[count + 1]);
+ final Snapshot s = dir.getSnapshot(components[count + 1]);
if (s == null) {
//snapshot not found
curNode = null;
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Sat Jul 12 02:24:40 2014
@@ -201,25 +201,28 @@ public class NameNode implements NameNod
};
private static final String USAGE = "Usage: java NameNode ["
- + StartupOption.BACKUP.getName() + "] | ["
- + StartupOption.CHECKPOINT.getName() + "] | ["
+ + StartupOption.BACKUP.getName() + "] | \n\t["
+ + StartupOption.CHECKPOINT.getName() + "] | \n\t["
+ StartupOption.FORMAT.getName() + " ["
+ StartupOption.CLUSTERID.getName() + " cid ] ["
+ StartupOption.FORCE.getName() + "] ["
- + StartupOption.NONINTERACTIVE.getName() + "] ] | ["
+ + StartupOption.NONINTERACTIVE.getName() + "] ] | \n\t["
+ StartupOption.UPGRADE.getName() +
" [" + StartupOption.CLUSTERID.getName() + " cid]" +
- " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | ["
- + StartupOption.ROLLBACK.getName() + "] | ["
+ " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | \n\t["
+ + StartupOption.ROLLBACK.getName() + "] | \n\t["
+ StartupOption.ROLLINGUPGRADE.getName() + " <"
+ RollingUpgradeStartupOption.DOWNGRADE.name().toLowerCase() + "|"
- + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | ["
- + StartupOption.FINALIZE.getName() + "] | ["
- + StartupOption.IMPORT.getName() + "] | ["
- + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | ["
- + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | ["
- + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName()
- + " ] ]";
+ + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | \n\t["
+ + StartupOption.FINALIZE.getName() + "] | \n\t["
+ + StartupOption.IMPORT.getName() + "] | \n\t["
+ + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | \n\t["
+ + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | \n\t["
+ + StartupOption.RECOVER.getName() + " [ "
+ + StartupOption.FORCE.getName() + "] ] | \n\t["
+ + StartupOption.METADATAVERSION.getName() + " ] "
+ + " ]";
+
public long getProtocolVersion(String protocol,
long clientVersion) throws IOException {
@@ -598,7 +601,8 @@ public class NameNode implements NameNod
pauseMonitor = new JvmPauseMonitor(conf);
pauseMonitor.start();
-
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
+
startCommonServices(conf);
}
@@ -830,7 +834,7 @@ public class NameNode implements NameNod
/** get FSImage */
@VisibleForTesting
public FSImage getFSImage() {
- return namesystem.dir.fsImage;
+ return namesystem.getFSImage();
}
/**
@@ -1140,7 +1144,7 @@ public class NameNode implements NameNod
return true;
}
}
- nsys.dir.fsImage.doRollback(nsys);
+ nsys.getFSImage().doRollback(nsys);
return false;
}
@@ -1265,6 +1269,8 @@ public class NameNode implements NameNod
"can't understand option \"" + args[i] + "\"");
}
}
+ } else if (StartupOption.METADATAVERSION.getName().equalsIgnoreCase(cmd)) {
+ startOpt = StartupOption.METADATAVERSION;
} else {
return null;
}
@@ -1317,6 +1323,21 @@ public class NameNode implements NameNod
}
}
+ /**
+ * Verify that configured directories exist, then print the metadata versions
+ * of the software and the image.
+ *
+ * @param conf configuration to use
+ * @throws IOException
+ */
+ private static boolean printMetadataVersion(Configuration conf)
+ throws IOException {
+ final FSImage fsImage = new FSImage(conf);
+ final FSNamesystem fs = new FSNamesystem(conf, fsImage, false);
+ return fsImage.recoverTransitionRead(
+ StartupOption.METADATAVERSION, fs, null);
+ }
+
public static NameNode createNameNode(String argv[], Configuration conf)
throws IOException {
LOG.info("createNameNode " + Arrays.asList(argv));
@@ -1381,6 +1402,11 @@ public class NameNode implements NameNod
NameNode.doRecovery(startOpt, conf);
return null;
}
+ case METADATAVERSION: {
+ printMetadataVersion(conf);
+ terminate(0);
+ return null; // avoid javac warning
+ }
default: {
DefaultMetricsSystem.initialize("NameNode");
return new NameNode(conf);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Sat Jul 12 02:24:40 2014
@@ -1046,6 +1046,11 @@ class NameNodeRpcServer implements Namen
boolean noStaleStorages = false;
for(StorageBlockReport r : reports) {
final BlockListAsLongs blocks = new BlockListAsLongs(r.getBlocks());
+ //
+ // BlockManager.processReport accumulates information of prior calls
+ // for the same node and storage, so the value returned by the last
+ // call of this loop is the final updated value for noStaleStorage.
+ //
noStaleStorages = bm.processReport(nodeReg, r.getStorage(), poolId, blocks);
metrics.incrStorageBlockReportOps();
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Sat Jul 12 02:24:40 2014
@@ -1064,7 +1064,7 @@ public class SecondaryNameNode implement
} finally {
dstNamesystem.writeUnlock();
}
- dstNamesystem.dir.imageLoadComplete();
+ dstNamesystem.imageLoadComplete();
}
// error simulation code for junit test
CheckpointFaultInjector.getInstance().duringMerge();
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java Sat Jul 12 02:24:40 2014
@@ -34,7 +34,8 @@ import com.google.common.collect.Lists;
* USER - extended user attributes: these can be assigned to files and
* directories to store arbitrary additional information. The access
* permissions for user attributes are defined by the file permission
- * bits.
+ * bits. For sticky directories, only the owner and privileged user can
+ * write attributes.
* <br>
* TRUSTED - trusted extended attributes: these are visible/accessible
* only to/by the super user.
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java Sat Jul 12 02:24:40 2014
@@ -98,7 +98,11 @@ public class NameNodeMetrics {
@Metric("GetImageServlet putImage")
MutableRate putImage;
- NameNodeMetrics(String processName, String sessionId, int[] intervals) {
+ JvmMetrics jvmMetrics = null;
+
+ NameNodeMetrics(String processName, String sessionId, int[] intervals,
+ final JvmMetrics jvmMetrics) {
+ this.jvmMetrics = jvmMetrics;
registry.tag(ProcessName, processName).tag(SessionId, sessionId);
final int len = intervals.length;
@@ -124,14 +128,19 @@ public class NameNodeMetrics {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
String processName = r.toString();
MetricsSystem ms = DefaultMetricsSystem.instance();
- JvmMetrics.create(processName, sessionId, ms);
+ JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals =
conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
- return ms.register(new NameNodeMetrics(processName, sessionId, intervals));
+ return ms.register(new NameNodeMetrics(processName, sessionId,
+ intervals, jm));
}
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java Sat Jul 12 02:24:40 2014
@@ -227,32 +227,34 @@ abstract class AbstractINodeDiffList<N e
D diff = getDiffById(snapshotId);
return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId();
}
-
- /**
- * Check if changes have happened between two snapshots.
- * @param earlier The snapshot taken earlier
- * @param later The snapshot taken later
- * @return Whether or not modifications (including diretory/file metadata
- * change, file creation/deletion under the directory) have happened
- * between snapshots.
- */
- final boolean changedBetweenSnapshots(Snapshot earlier, Snapshot later) {
+
+ final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) {
+ Snapshot earlier = from;
+ Snapshot later = to;
+ if (Snapshot.ID_COMPARATOR.compare(from, to) > 0) {
+ earlier = to;
+ later = from;
+ }
+
final int size = diffs.size();
int earlierDiffIndex = Collections.binarySearch(diffs, earlier.getId());
+ int laterDiffIndex = later == null ? size : Collections
+ .binarySearch(diffs, later.getId());
if (-earlierDiffIndex - 1 == size) {
// if the earlierSnapshot is after the latest SnapshotDiff stored in
// diffs, no modification happened after the earlierSnapshot
- return false;
+ return null;
}
- if (later != null) {
- int laterDiffIndex = Collections.binarySearch(diffs, later.getId());
- if (laterDiffIndex == -1 || laterDiffIndex == 0) {
- // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or
- // before it, no modification happened before the laterSnapshot
- return false;
- }
+ if (laterDiffIndex == -1 || laterDiffIndex == 0) {
+ // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or
+ // before it, no modification happened before the laterSnapshot
+ return null;
}
- return true;
+ earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
+ : earlierDiffIndex;
+ laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
+ : laterDiffIndex;
+ return new int[]{earlierDiffIndex, laterDiffIndex};
}
/**
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java Sat Jul 12 02:24:40 2014
@@ -20,8 +20,6 @@ package org.apache.hadoop.hdfs.server.na
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
@@ -30,8 +28,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
@@ -52,7 +48,9 @@ import org.apache.hadoop.hdfs.util.ReadO
import com.google.common.base.Preconditions;
/**
- * Feature for directory with snapshot-related information.
+ * Feature used to store and process the snapshot diff information for a
+ * directory. In particular, it contains a directory diff list recording changes
+ * made to the directory and its children for each snapshot.
*/
@InterfaceAudience.Private
public class DirectoryWithSnapshotFeature implements INode.Feature {
@@ -161,59 +159,6 @@ public class DirectoryWithSnapshotFeatur
}
}
}
-
- /**
- * Interpret the diff and generate a list of {@link DiffReportEntry}.
- * @param parentPath The relative path of the parent.
- * @param fromEarlier True indicates {@code diff=later-earlier},
- * False indicates {@code diff=earlier-later}
- * @return A list of {@link DiffReportEntry} as the diff report.
- */
- public List<DiffReportEntry> generateReport(byte[][] parentPath,
- boolean fromEarlier) {
- List<DiffReportEntry> cList = new ArrayList<DiffReportEntry>();
- List<DiffReportEntry> dList = new ArrayList<DiffReportEntry>();
- int c = 0, d = 0;
- List<INode> created = getList(ListType.CREATED);
- List<INode> deleted = getList(ListType.DELETED);
- byte[][] fullPath = new byte[parentPath.length + 1][];
- System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
- for (; c < created.size() && d < deleted.size(); ) {
- INode cnode = created.get(c);
- INode dnode = deleted.get(d);
- if (cnode.compareTo(dnode.getLocalNameBytes()) == 0) {
- fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
- // must be the case: delete first and then create an inode with the
- // same name
- cList.add(new DiffReportEntry(DiffType.CREATE, fullPath));
- dList.add(new DiffReportEntry(DiffType.DELETE, fullPath));
- c++;
- d++;
- } else if (cnode.compareTo(dnode.getLocalNameBytes()) < 0) {
- fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
- cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
- : DiffType.DELETE, fullPath));
- c++;
- } else {
- fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
- dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
- : DiffType.CREATE, fullPath));
- d++;
- }
- }
- for (; d < deleted.size(); d++) {
- fullPath[fullPath.length - 1] = deleted.get(d).getLocalNameBytes();
- dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
- : DiffType.CREATE, fullPath));
- }
- for (; c < created.size(); c++) {
- fullPath[fullPath.length - 1] = created.get(c).getLocalNameBytes();
- cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
- : DiffType.DELETE, fullPath));
- }
- dList.addAll(cList);
- return dList;
- }
}
/**
@@ -724,34 +669,21 @@ public class DirectoryWithSnapshotFeatur
*/
boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot,
Snapshot toSnapshot, ChildrenDiff diff, INodeDirectory currentINode) {
- Snapshot earlier = fromSnapshot;
- Snapshot later = toSnapshot;
- if (Snapshot.ID_COMPARATOR.compare(fromSnapshot, toSnapshot) > 0) {
- earlier = toSnapshot;
- later = fromSnapshot;
- }
-
- boolean modified = diffs.changedBetweenSnapshots(earlier, later);
- if (!modified) {
+ int[] diffIndexPair = diffs.changedBetweenSnapshots(fromSnapshot,
+ toSnapshot);
+ if (diffIndexPair == null) {
return false;
}
-
- final List<DirectoryDiff> difflist = diffs.asList();
- final int size = difflist.size();
- int earlierDiffIndex = Collections.binarySearch(difflist, earlier.getId());
- int laterDiffIndex = later == null ? size : Collections
- .binarySearch(difflist, later.getId());
- earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
- : earlierDiffIndex;
- laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
- : laterDiffIndex;
+ int earlierDiffIndex = diffIndexPair[0];
+ int laterDiffIndex = diffIndexPair[1];
boolean dirMetadataChanged = false;
INodeDirectoryAttributes dirCopy = null;
+ List<DirectoryDiff> difflist = diffs.asList();
for (int i = earlierDiffIndex; i < laterDiffIndex; i++) {
DirectoryDiff sdiff = difflist.get(i);
diff.combinePosterior(sdiff.diff, null);
- if (dirMetadataChanged == false && sdiff.snapshotINode != null) {
+ if (!dirMetadataChanged && sdiff.snapshotINode != null) {
if (dirCopy == null) {
dirCopy = sdiff.snapshotINode;
} else if (!dirCopy.metadataEquals(sdiff.snapshotINode)) {
@@ -763,7 +695,7 @@ public class DirectoryWithSnapshotFeatur
if (!diff.isEmpty() || dirMetadataChanged) {
return true;
} else if (dirCopy != null) {
- for (int i = laterDiffIndex; i < size; i++) {
+ for (int i = laterDiffIndex; i < difflist.size(); i++) {
if (!dirCopy.metadataEquals(difflist.get(i).snapshotINode)) {
return true;
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java?rev=1609878&r1=1609877&r2=1609878&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java Sat Jul 12 02:24:40 2014
@@ -127,9 +127,8 @@ public class FSImageFormatPBSnapshot {
}
/**
- * Load the snapshots section from fsimage. Also convert snapshottable
- * directories into {@link INodeDirectorySnapshottable}.
- *
+ * Load the snapshots section from fsimage. Also add snapshottable feature
+ * to snapshottable directories.
*/
public void loadSnapshotSection(InputStream in) throws IOException {
SnapshotManager sm = fsn.getSnapshotManager();
@@ -139,16 +138,13 @@ public class FSImageFormatPBSnapshot {
sm.setSnapshotCounter(section.getSnapshotCounter());
for (long sdirId : section.getSnapshottableDirList()) {
INodeDirectory dir = fsDir.getInode(sdirId).asDirectory();
- final INodeDirectorySnapshottable sdir;
if (!dir.isSnapshottable()) {
- sdir = new INodeDirectorySnapshottable(dir);
- fsDir.addToInodeMap(sdir);
+ dir.addSnapshottableFeature();
} else {
// dir is root, and admin set root to snapshottable before
- sdir = (INodeDirectorySnapshottable) dir;
- sdir.setSnapshotQuota(INodeDirectorySnapshottable.SNAPSHOT_LIMIT);
+ dir.setSnapshotQuota(DirectorySnapshottableFeature.SNAPSHOT_LIMIT);
}
- sm.addSnapshottable(sdir);
+ sm.addSnapshottable(dir);
}
loadSnapshots(in, snum);
}
@@ -160,12 +156,11 @@ public class FSImageFormatPBSnapshot {
INodeDirectory root = loadINodeDirectory(pbs.getRoot(),
parent.getLoaderContext());
int sid = pbs.getSnapshotId();
- INodeDirectorySnapshottable parent = (INodeDirectorySnapshottable) fsDir
- .getInode(root.getId()).asDirectory();
+ INodeDirectory parent = fsDir.getInode(root.getId()).asDirectory();
Snapshot snapshot = new Snapshot(sid, root, parent);
// add the snapshot to parent, since we follow the sequence of
// snapshotsByNames when saving, we do not need to sort when loading
- parent.addSnapshot(snapshot);
+ parent.getDirectorySnapshottableFeature().addSnapshot(snapshot);
snapshotMap.put(sid, snapshot);
}
}
@@ -373,14 +368,15 @@ public class FSImageFormatPBSnapshot {
.setSnapshotCounter(sm.getSnapshotCounter())
.setNumSnapshots(sm.getNumSnapshots());
- INodeDirectorySnapshottable[] snapshottables = sm.getSnapshottableDirs();
- for (INodeDirectorySnapshottable sdir : snapshottables) {
+ INodeDirectory[] snapshottables = sm.getSnapshottableDirs();
+ for (INodeDirectory sdir : snapshottables) {
b.addSnapshottableDir(sdir.getId());
}
b.build().writeDelimitedTo(out);
int i = 0;
- for(INodeDirectorySnapshottable sdir : snapshottables) {
- for(Snapshot s : sdir.getSnapshotsByNames()) {
+ for(INodeDirectory sdir : snapshottables) {
+ for (Snapshot s : sdir.getDirectorySnapshottableFeature()
+ .getSnapshotList()) {
Root sroot = s.getRoot();
SnapshotSection.Snapshot.Builder sb = SnapshotSection.Snapshot
.newBuilder().setSnapshotId(s.getId());