You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2014/07/15 23:10:29 UTC
svn commit: r1610853 [2/3] - in
/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs: ./
src/contrib/bkjournal/
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/contrib/bkjournal/src/test/java/org/apache/had...
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Tue Jul 15 21:10:24 2014
@@ -81,7 +81,7 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
import org.apache.hadoop.hdfs.util.ByteArray;
@@ -101,7 +101,7 @@ import com.google.common.collect.Lists;
**/
@InterfaceAudience.Private
public class FSDirectory implements Closeable {
- private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
+ private static INodeDirectory createRoot(FSNamesystem namesystem) {
final INodeDirectory r = new INodeDirectory(
INodeId.ROOT_INODE_ID,
INodeDirectory.ROOT_NAME,
@@ -110,9 +110,9 @@ public class FSDirectory implements Clos
r.addDirectoryWithQuotaFeature(
DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA,
DirectoryWithQuotaFeature.DEFAULT_DISKSPACE_QUOTA);
- final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(r);
- s.setSnapshotQuota(0);
- return s;
+ r.addSnapshottableFeature();
+ r.setSnapshotQuota(0);
+ return r;
}
@VisibleForTesting
@@ -606,8 +606,7 @@ public class FSDirectory implements Clos
ezManager.checkMoveValidity(srcIIP, dstIIP, src);
final INode dstInode = dstIIP.getLastINode();
- List<INodeDirectorySnapshottable> snapshottableDirs =
- new ArrayList<INodeDirectorySnapshottable>();
+ List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
if (dstInode != null) { // Destination exists
validateRenameOverwrite(src, dst, overwrite, srcInode, dstInode);
checkSnapshot(dstInode, snapshottableDirs);
@@ -676,7 +675,7 @@ public class FSDirectory implements Clos
dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes,
true).get(Quota.NAMESPACE);
getFSNamesystem().removePathAndBlocks(src, collectedBlocks,
- removedINodes);
+ removedINodes, false);
}
}
@@ -1133,8 +1132,7 @@ public class FSDirectory implements Clos
if (!deleteAllowed(inodesInPath, src) ) {
filesRemoved = -1;
} else {
- List<INodeDirectorySnapshottable> snapshottableDirs =
- new ArrayList<INodeDirectorySnapshottable>();
+ List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
removedINodes, mtime);
@@ -1204,8 +1202,7 @@ public class FSDirectory implements Clos
normalizePath(src), false);
long filesRemoved = -1;
if (deleteAllowed(inodesInPath, src)) {
- List<INodeDirectorySnapshottable> snapshottableDirs =
- new ArrayList<INodeDirectorySnapshottable>();
+ List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
removedINodes, mtime);
@@ -1214,7 +1211,7 @@ public class FSDirectory implements Clos
if (filesRemoved >= 0) {
getFSNamesystem().removePathAndBlocks(src, collectedBlocks,
- removedINodes);
+ removedINodes, false);
}
}
@@ -1281,19 +1278,20 @@ public class FSDirectory implements Clos
* but do not have snapshots yet
*/
private static void checkSnapshot(INode target,
- List<INodeDirectorySnapshottable> snapshottableDirs) throws SnapshotException {
+ List<INodeDirectory> snapshottableDirs) throws SnapshotException {
if (target.isDirectory()) {
INodeDirectory targetDir = target.asDirectory();
- if (targetDir.isSnapshottable()) {
- INodeDirectorySnapshottable ssTargetDir =
- (INodeDirectorySnapshottable) targetDir;
- if (ssTargetDir.getNumSnapshots() > 0) {
- throw new SnapshotException("The directory " + ssTargetDir.getFullPathName()
- + " cannot be deleted since " + ssTargetDir.getFullPathName()
+ DirectorySnapshottableFeature sf = targetDir
+ .getDirectorySnapshottableFeature();
+ if (sf != null) {
+ if (sf.getNumSnapshots() > 0) {
+ String fullPath = targetDir.getFullPathName();
+ throw new SnapshotException("The directory " + fullPath
+ + " cannot be deleted since " + fullPath
+ " is snapshottable and already has snapshots");
} else {
if (snapshottableDirs != null) {
- snapshottableDirs.add(ssTargetDir);
+ snapshottableDirs.add(targetDir);
}
}
}
@@ -1380,14 +1378,18 @@ public class FSDirectory implements Clos
Preconditions.checkArgument(
src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
"%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
-
+
final String dirPath = normalizePath(src.substring(0,
src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
final INode node = this.getINode(dirPath);
- final INodeDirectorySnapshottable dirNode = INodeDirectorySnapshottable
- .valueOf(node, dirPath);
- final ReadOnlyList<Snapshot> snapshots = dirNode.getSnapshotList();
+ final INodeDirectory dirNode = INodeDirectory.valueOf(node, dirPath);
+ final DirectorySnapshottableFeature sf = dirNode.getDirectorySnapshottableFeature();
+ if (sf == null) {
+ throw new SnapshotException(
+ "Directory is not a snapshottable directory: " + dirPath);
+ }
+ final ReadOnlyList<Snapshot> snapshots = sf.getSnapshotList();
int skipSize = ReadOnlyList.Util.binarySearch(snapshots, startAfter);
skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1;
int numOfListing = Math.min(snapshots.size() - skipSize, this.lsLimit);
@@ -1451,9 +1453,8 @@ public class FSDirectory implements Clos
src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
final INode node = this.getINode(dirPath);
- if (node != null
- && node.isDirectory()
- && node.asDirectory() instanceof INodeDirectorySnapshottable) {
+ if (node != null && node.isDirectory()
+ && node.asDirectory().isSnapshottable()) {
return node;
}
return null;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Tue Jul 15 21:10:24 2014
@@ -58,7 +58,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
@@ -73,8 +72,8 @@ import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.StringUtils;
-import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
/**
* Contains inner classes for reading or writing the on-disk format for
@@ -554,21 +553,17 @@ public class FSImageFormat {
if (!toLoadSubtree) {
return;
}
-
+
// Step 2. Load snapshots if parent is snapshottable
int numSnapshots = in.readInt();
if (numSnapshots >= 0) {
- final INodeDirectorySnapshottable snapshottableParent
- = INodeDirectorySnapshottable.valueOf(parent, parent.getLocalName());
// load snapshots and snapshotQuota
- SnapshotFSImageFormat.loadSnapshotList(snapshottableParent,
- numSnapshots, in, this);
- if (snapshottableParent.getSnapshotQuota() > 0) {
+ SnapshotFSImageFormat.loadSnapshotList(parent, numSnapshots, in, this);
+ if (parent.getDirectorySnapshottableFeature().getSnapshotQuota() > 0) {
// add the directory to the snapshottable directory list in
// SnapshotManager. Note that we only add root when its snapshot quota
// is positive.
- this.namesystem.getSnapshotManager().addSnapshottable(
- snapshottableParent);
+ this.namesystem.getSnapshotManager().addSnapshottable(parent);
}
}
@@ -820,7 +815,10 @@ public class FSImageFormat {
if (withSnapshot) {
dir.addSnapshotFeature(null);
}
- return snapshottable ? new INodeDirectorySnapshottable(dir) : dir;
+ if (snapshottable) {
+ dir.addSnapshottableFeature();
+ }
+ return dir;
} else if (numBlocks == -2) {
//symlink
@@ -1367,10 +1365,8 @@ public class FSImageFormat {
// 2. Write INodeDirectorySnapshottable#snapshotsByNames to record all
// Snapshots
- if (current instanceof INodeDirectorySnapshottable) {
- INodeDirectorySnapshottable snapshottableNode =
- (INodeDirectorySnapshottable) current;
- SnapshotFSImageFormat.saveSnapshots(snapshottableNode, out);
+ if (current.isDirectory() && current.asDirectory().isSnapshottable()) {
+ SnapshotFSImageFormat.saveSnapshots(current.asDirectory(), out);
} else {
out.writeInt(-1); // # of snapshots
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Tue Jul 15 21:10:24 2014
@@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.XMLUtils;
@@ -241,7 +240,7 @@ public class FSImageSerialization {
writeQuota(node.getQuotaCounts(), out);
- if (node instanceof INodeDirectorySnapshottable) {
+ if (node.isSnapshottable()) {
out.writeBoolean(true);
} else {
out.writeBoolean(false);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Jul 15 21:10:24 2014
@@ -229,7 +229,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -3000,9 +2999,10 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INodeFile pendingFile = analyzeFileState(
+ FileState fileState = analyzeFileState(
src, fileId, clientName, previous, onRetryBlock);
- src = pendingFile.getFullPathName();
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
// This is a retry. Just return the last block if having locations.
@@ -3038,8 +3038,10 @@ public class FSNamesystem implements Nam
// Run the full analysis again, since things could have changed
// while chooseTarget() was executing.
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INodeFile pendingFile =
+ FileState fileState =
analyzeFileState(src, fileId, clientName, previous, onRetryBlock);
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null) {
if (onRetryBlock[0].getLocations().length > 0) {
@@ -3075,7 +3077,17 @@ public class FSNamesystem implements Nam
return makeLocatedBlock(newBlock, targets, offset);
}
- INodeFile analyzeFileState(String src,
+ static class FileState {
+ public final INodeFile inode;
+ public final String path;
+
+ public FileState(INodeFile inode, String fullPath) {
+ this.inode = inode;
+ this.path = fullPath;
+ }
+ }
+
+ FileState analyzeFileState(String src,
long fileId,
String clientName,
ExtendedBlock previous,
@@ -3163,7 +3175,7 @@ public class FSNamesystem implements Nam
onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
offset);
- return pendingFile;
+ return new FileState(pendingFile, src);
} else {
// Case 3
throw new IOException("Cannot allocate block in " + src + ": " +
@@ -3176,7 +3188,7 @@ public class FSNamesystem implements Nam
if (!checkFileProgress(pendingFile, false)) {
throw new NotReplicatedYetException("Not replicated yet: " + src);
}
- return pendingFile;
+ return new FileState(pendingFile, src);
}
LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
@@ -3324,10 +3336,9 @@ public class FSNamesystem implements Nam
: "Holder " + holder + " does not have any open files."));
}
// No further modification is allowed on a deleted file.
- // A file is considered deleted, if it has no parent or is marked
+ // A file is considered deleted, if it is not in the inodeMap or is marked
// as deleted in the snapshot feature.
- if (file.getParent() == null || (file.isWithSnapshot() &&
- file.getFileWithSnapshotFeature().isCurrentFileDeleted())) {
+ if (isFileDeleted(file)) {
throw new FileNotFoundException(src);
}
String clientName = file.getFileUnderConstructionFeature().getClientName();
@@ -3762,7 +3773,7 @@ public class FSNamesystem implements Nam
getEditLog().logDelete(src, mtime, logRetryCache);
incrDeletedFileCount(filesRemoved);
// Blocks/INodes will be handled later
- removePathAndBlocks(src, null, null);
+ removePathAndBlocks(src, null, removedINodes, true);
ret = true;
} finally {
writeUnlock();
@@ -3771,13 +3782,6 @@ public class FSNamesystem implements Nam
removeBlocks(collectedBlocks); // Incremental deletion of blocks
collectedBlocks.clear();
- dir.writeLock();
- try {
- dir.removeFromInodeMap(removedINodes);
- } finally {
- dir.writeUnlock();
- }
- removedINodes.clear();
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "
+ src +" is removed");
@@ -3815,14 +3819,24 @@ public class FSNamesystem implements Nam
* @param blocks Containing the list of blocks to be deleted from blocksMap
* @param removedINodes Containing the list of inodes to be removed from
* inodesMap
+ * @param acquireINodeMapLock Whether to acquire the lock for inode removal
*/
void removePathAndBlocks(String src, BlocksMapUpdateInfo blocks,
- List<INode> removedINodes) {
+ List<INode> removedINodes, final boolean acquireINodeMapLock) {
assert hasWriteLock();
leaseManager.removeLeaseWithPrefixPath(src);
// remove inodes from inodesMap
if (removedINodes != null) {
- dir.removeFromInodeMap(removedINodes);
+ if (acquireINodeMapLock) {
+ dir.writeLock();
+ }
+ try {
+ dir.removeFromInodeMap(removedINodes);
+ } finally {
+ if (acquireINodeMapLock) {
+ dir.writeUnlock();
+ }
+ }
removedINodes.clear();
}
if (blocks == null) {
@@ -6472,6 +6486,16 @@ public class FSNamesystem implements Nam
return blockId;
}
+ private boolean isFileDeleted(INodeFile file) {
+ // Not in the inodeMap or in the snapshot but marked deleted.
+ if (dir.getInode(file.getId()) == null ||
+ file.getParent() == null || (file.isWithSnapshot() &&
+ file.getFileWithSnapshotFeature().isCurrentFileDeleted())) {
+ return true;
+ }
+ return false;
+ }
+
private INodeFile checkUCBlock(ExtendedBlock block,
String clientName) throws IOException {
assert hasWriteLock();
@@ -6488,7 +6512,7 @@ public class FSNamesystem implements Nam
// check file inode
final INodeFile file = ((INode)storedBlock.getBlockCollection()).asFile();
- if (file == null || !file.isUnderConstruction()) {
+ if (file == null || !file.isUnderConstruction() || isFileDeleted(file)) {
throw new IOException("The file " + storedBlock +
" belonged to does not exist or it is not under construction.");
}
@@ -7816,7 +7840,7 @@ public class FSNamesystem implements Nam
* Remove a list of INodeDirectorySnapshottable from the SnapshotManager
* @param toRemove the list of INodeDirectorySnapshottable to be removed
*/
- void removeSnapshottableDirs(List<INodeDirectorySnapshottable> toRemove) {
+ void removeSnapshottableDirs(List<INodeDirectory> toRemove) {
if (snapshotManager != null) {
snapshotManager.removeSnapshottable(toRemove);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java Tue Jul 15 21:10:24 2014
@@ -97,14 +97,12 @@ public interface INodeAttributes {
@Override
public final String getUserName() {
- final int n = (int)PermissionStatusFormat.USER.retrieve(permission);
- return SerialNumberManager.INSTANCE.getUser(n);
+ return PermissionStatusFormat.getUser(permission);
}
@Override
public final String getGroupName() {
- final int n = (int)PermissionStatusFormat.GROUP.retrieve(permission);
- return SerialNumberManager.INSTANCE.getGroup(n);
+ return PermissionStatusFormat.getGroup(permission);
}
@Override
@@ -114,7 +112,7 @@ public interface INodeAttributes {
@Override
public final short getFsPermissionShort() {
- return (short)PermissionStatusFormat.MODE.retrieve(permission);
+ return PermissionStatusFormat.getMode(permission);
}
@Override
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Tue Jul 15 21:10:24 2014
@@ -29,10 +29,11 @@ import org.apache.hadoop.fs.PathIsNotDir
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.Diff.ListType;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -102,11 +103,6 @@ public class INodeDirectory extends INod
return this;
}
- /** Is this a snapshottable directory? */
- public boolean isSnapshottable() {
- return false;
- }
-
void setQuota(long nsQuota, long dsQuota) {
DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
if (quota != null) {
@@ -186,7 +182,7 @@ public class INodeDirectory extends INod
public final boolean isWithSnapshot() {
return getDirectoryWithSnapshotFeature() != null;
}
-
+
public DirectoryDiffList getDiffs() {
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
return sf != null ? sf.getDiffs() : null;
@@ -204,50 +200,71 @@ public class INodeDirectory extends INod
return super.toDetailString() + (sf == null ? "" : ", " + sf.getDiffs());
}
- /** Replace itself with an {@link INodeDirectorySnapshottable}. */
- public INodeDirectorySnapshottable replaceSelf4INodeDirectorySnapshottable(
- int latestSnapshotId, final INodeMap inodeMap)
- throws QuotaExceededException {
- Preconditions.checkState(!(this instanceof INodeDirectorySnapshottable),
- "this is already an INodeDirectorySnapshottable, this=%s", this);
- final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(this);
- replaceSelf(s, inodeMap).getDirectoryWithSnapshotFeature().getDiffs()
- .saveSelf2Snapshot(latestSnapshotId, s, this);
- return s;
- }
-
- /** Replace itself with {@link INodeDirectory}. */
- public INodeDirectory replaceSelf4INodeDirectory(final INodeMap inodeMap) {
- Preconditions.checkState(getClass() != INodeDirectory.class,
- "the class is already INodeDirectory, this=%s", this);
- return replaceSelf(new INodeDirectory(this, true, this.getFeatures()),
- inodeMap);
+ public DirectorySnapshottableFeature getDirectorySnapshottableFeature() {
+ return getFeature(DirectorySnapshottableFeature.class);
}
- /** Replace itself with the given directory. */
- private final <N extends INodeDirectory> N replaceSelf(final N newDir,
- final INodeMap inodeMap) {
- final INodeReference ref = getParentReference();
- if (ref != null) {
- ref.setReferredINode(newDir);
- if (inodeMap != null) {
- inodeMap.put(newDir);
- }
- } else {
- final INodeDirectory parent = getParent();
- Preconditions.checkArgument(parent != null, "parent is null, this=%s", this);
- parent.replaceChild(this, newDir, inodeMap);
+ public boolean isSnapshottable() {
+ return getDirectorySnapshottableFeature() != null;
+ }
+
+ public Snapshot getSnapshot(byte[] snapshotName) {
+ return getDirectorySnapshottableFeature().getSnapshot(snapshotName);
+ }
+
+ public void setSnapshotQuota(int snapshotQuota) {
+ getDirectorySnapshottableFeature().setSnapshotQuota(snapshotQuota);
+ }
+
+ public Snapshot addSnapshot(int id, String name) throws SnapshotException,
+ QuotaExceededException {
+ return getDirectorySnapshottableFeature().addSnapshot(this, id, name);
+ }
+
+ public Snapshot removeSnapshot(String snapshotName,
+ BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
+ throws SnapshotException {
+ return getDirectorySnapshottableFeature().removeSnapshot(this,
+ snapshotName, collectedBlocks, removedINodes);
+ }
+
+ public void renameSnapshot(String path, String oldName, String newName)
+ throws SnapshotException {
+ getDirectorySnapshottableFeature().renameSnapshot(path, oldName, newName);
+ }
+
+ /** add DirectorySnapshottableFeature */
+ public void addSnapshottableFeature() {
+ Preconditions.checkState(!isSnapshottable(),
+ "this is already snapshottable, this=%s", this);
+ DirectoryWithSnapshotFeature s = this.getDirectoryWithSnapshotFeature();
+ final DirectorySnapshottableFeature snapshottable =
+ new DirectorySnapshottableFeature(s);
+ if (s != null) {
+ this.removeFeature(s);
}
- clear();
- return newDir;
+ this.addFeature(snapshottable);
}
-
+
+ /** remove DirectorySnapshottableFeature */
+ public void removeSnapshottableFeature() {
+ DirectorySnapshottableFeature s = getDirectorySnapshottableFeature();
+ Preconditions.checkState(s != null,
+ "The dir does not have snapshottable feature: this=%s", this);
+ this.removeFeature(s);
+ if (s.getDiffs().asList().size() > 0) {
+ // add a DirectoryWithSnapshotFeature back
+ DirectoryWithSnapshotFeature sf = new DirectoryWithSnapshotFeature(
+ s.getDiffs());
+ addFeature(sf);
+ }
+ }
+
/**
* Replace the given child with a new child. Note that we no longer need to
* replace an normal INodeDirectory or INodeFile into an
* INodeDirectoryWithSnapshot or INodeFileUnderConstruction. The only cases
- * for child replacement is for {@link INodeDirectorySnapshottable} and
- * reference nodes.
+ * for child replacement is for reference nodes.
*/
public void replaceChild(INode oldChild, final INode newChild,
final INodeMap inodeMap) {
@@ -822,6 +839,11 @@ public class INodeDirectory extends INod
};
}
});
+
+ final DirectorySnapshottableFeature s = getDirectorySnapshottableFeature();
+ if (s != null) {
+ s.dumpTreeRecursively(this, out, prefix, snapshot);
+ }
}
/**
@@ -830,7 +852,7 @@ public class INodeDirectory extends INod
* @param subs The subtrees.
*/
@VisibleForTesting
- protected static void dumpTreeRecursively(PrintWriter out,
+ public static void dumpTreeRecursively(PrintWriter out,
StringBuilder prefix, Iterable<SnapshotAndINode> subs) {
if (subs != null) {
for(final Iterator<SnapshotAndINode> i = subs.iterator(); i.hasNext();) {
@@ -843,7 +865,7 @@ public class INodeDirectory extends INod
}
/** A pair of Snapshot and INode objects. */
- protected static class SnapshotAndINode {
+ public static class SnapshotAndINode {
public final int snapshotId;
public final INode inode;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Tue Jul 15 21:10:24 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -71,37 +72,29 @@ public class INodeFile extends INodeWith
}
/** Format: [16 bits for replication][48 bits for PreferredBlockSize] */
- static class HeaderFormat {
- /** Number of bits for Block size */
- static final int BLOCKBITS = 48;
- /** Header mask 64-bit representation */
- static final long HEADERMASK = 0xffffL << BLOCKBITS;
- static final long MAX_BLOCK_SIZE = ~HEADERMASK;
-
- static short getReplication(long header) {
- return (short) ((header & HEADERMASK) >> BLOCKBITS);
+ static enum HeaderFormat {
+ PREFERRED_BLOCK_SIZE(null, 48, 1),
+ REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 16, 1);
+
+ private final LongBitFormat BITS;
+
+ private HeaderFormat(LongBitFormat previous, int length, long min) {
+ BITS = new LongBitFormat(name(), previous, length, min);
}
- static long combineReplication(long header, short replication) {
- if (replication <= 0) {
- throw new IllegalArgumentException(
- "Unexpected value for the replication: " + replication);
- }
- return ((long)replication << BLOCKBITS) | (header & MAX_BLOCK_SIZE);
+ static short getReplication(long header) {
+ return (short)REPLICATION.BITS.retrieve(header);
}
-
+
static long getPreferredBlockSize(long header) {
- return header & MAX_BLOCK_SIZE;
+ return PREFERRED_BLOCK_SIZE.BITS.retrieve(header);
}
- static long combinePreferredBlockSize(long header, long blockSize) {
- if (blockSize < 0) {
- throw new IllegalArgumentException("Block size < 0: " + blockSize);
- } else if (blockSize > MAX_BLOCK_SIZE) {
- throw new IllegalArgumentException("Block size = " + blockSize
- + " > MAX_BLOCK_SIZE = " + MAX_BLOCK_SIZE);
- }
- return (header & HEADERMASK) | (blockSize & MAX_BLOCK_SIZE);
+ static long toLong(long preferredBlockSize, short replication) {
+ long h = 0;
+ h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
+ h = REPLICATION.BITS.combine(replication, h);
+ return h;
}
}
@@ -113,8 +106,7 @@ public class INodeFile extends INodeWith
long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize) {
super(id, name, permissions, mtime, atime);
- header = HeaderFormat.combineReplication(header, replication);
- header = HeaderFormat.combinePreferredBlockSize(header, preferredBlockSize);
+ header = HeaderFormat.toLong(preferredBlockSize, replication);
this.blocks = blklist;
}
@@ -347,7 +339,7 @@ public class INodeFile extends INodeWith
/** Set the replication factor of this file. */
public final void setFileReplication(short replication) {
- header = HeaderFormat.combineReplication(header, replication);
+ header = HeaderFormat.REPLICATION.BITS.combine(replication, header);
}
/** Set the replication factor of this file. */
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java Tue Jul 15 21:10:24 2014
@@ -48,9 +48,7 @@ public interface INodeFileAttributes ext
short replication, long preferredBlockSize, XAttrFeature xAttrsFeature) {
super(name, permissions, aclFeature, modificationTime, accessTime,
xAttrsFeature);
-
- final long h = HeaderFormat.combineReplication(0L, replication);
- header = HeaderFormat.combinePreferredBlockSize(h, preferredBlockSize);
+ header = HeaderFormat.toLong(preferredBlockSize, replication);
}
public SnapshotCopy(INodeFile file) {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java Tue Jul 15 21:10:24 2014
@@ -21,9 +21,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.namenode.INode.Feature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
-import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
import com.google.common.base.Preconditions;
@@ -36,26 +35,28 @@ import com.google.common.base.Preconditi
public abstract class INodeWithAdditionalFields extends INode
implements LinkedElement {
static enum PermissionStatusFormat {
- MODE(0, 16),
- GROUP(MODE.OFFSET + MODE.LENGTH, 25),
- USER(GROUP.OFFSET + GROUP.LENGTH, 23);
+ MODE(null, 16),
+ GROUP(MODE.BITS, 25),
+ USER(GROUP.BITS, 23);
- final int OFFSET;
- final int LENGTH; //bit length
- final long MASK;
+ final LongBitFormat BITS;
- PermissionStatusFormat(int offset, int length) {
- OFFSET = offset;
- LENGTH = length;
- MASK = ((-1L) >>> (64 - LENGTH)) << OFFSET;
+ private PermissionStatusFormat(LongBitFormat previous, int length) {
+ BITS = new LongBitFormat(name(), previous, length, 0);
}
- long retrieve(long record) {
- return (record & MASK) >>> OFFSET;
+ static String getUser(long permission) {
+ final int n = (int)USER.BITS.retrieve(permission);
+ return SerialNumberManager.INSTANCE.getUser(n);
}
- long combine(long bits, long record) {
- return (record & ~MASK) | (bits << OFFSET);
+ static String getGroup(long permission) {
+ final int n = (int)GROUP.BITS.retrieve(permission);
+ return SerialNumberManager.INSTANCE.getGroup(n);
+ }
+
+ static short getMode(long permission) {
+ return (short)MODE.BITS.retrieve(permission);
}
/** Encode the {@link PermissionStatus} to a long. */
@@ -63,12 +64,12 @@ public abstract class INodeWithAdditiona
long permission = 0L;
final int user = SerialNumberManager.INSTANCE.getUserSerialNumber(
ps.getUserName());
- permission = USER.combine(user, permission);
+ permission = USER.BITS.combine(user, permission);
final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber(
ps.getGroupName());
- permission = GROUP.combine(group, permission);
+ permission = GROUP.BITS.combine(group, permission);
final int mode = ps.getPermission().toShort();
- permission = MODE.combine(mode, permission);
+ permission = MODE.BITS.combine(mode, permission);
return permission;
}
}
@@ -162,7 +163,7 @@ public abstract class INodeWithAdditiona
}
private final void updatePermissionStatus(PermissionStatusFormat f, long n) {
- this.permission = f.combine(n, permission);
+ this.permission = f.BITS.combine(n, permission);
}
@Override
@@ -170,9 +171,7 @@ public abstract class INodeWithAdditiona
if (snapshotId != Snapshot.CURRENT_STATE_ID) {
return getSnapshotINode(snapshotId).getUserName();
}
-
- int n = (int)PermissionStatusFormat.USER.retrieve(permission);
- return SerialNumberManager.INSTANCE.getUser(n);
+ return PermissionStatusFormat.getUser(permission);
}
@Override
@@ -186,9 +185,7 @@ public abstract class INodeWithAdditiona
if (snapshotId != Snapshot.CURRENT_STATE_ID) {
return getSnapshotINode(snapshotId).getGroupName();
}
-
- int n = (int)PermissionStatusFormat.GROUP.retrieve(permission);
- return SerialNumberManager.INSTANCE.getGroup(n);
+ return PermissionStatusFormat.getGroup(permission);
}
@Override
@@ -208,7 +205,7 @@ public abstract class INodeWithAdditiona
@Override
public final short getFsPermissionShort() {
- return (short)PermissionStatusFormat.MODE.retrieve(permission);
+ return PermissionStatusFormat.getMode(permission);
}
@Override
void setPermission(FsPermission permission) {
@@ -318,8 +315,9 @@ public abstract class INodeWithAdditiona
}
protected <T extends Feature> T getFeature(Class<? extends Feature> clazz) {
+ Preconditions.checkArgument(clazz != null);
for (Feature f : features) {
- if (f.getClass() == clazz) {
+ if (clazz.isAssignableFrom(f.getClass())) {
@SuppressWarnings("unchecked")
T ret = (T) f;
return ret;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java Tue Jul 15 21:10:24 2014
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import com.google.common.base.Preconditions;
@@ -208,8 +207,7 @@ public class INodesInPath {
final byte[] childName = components[count + 1];
// check if the next byte[] in components is for ".snapshot"
- if (isDotSnapshotDir(childName)
- && isDir && dir instanceof INodeDirectorySnapshottable) {
+ if (isDotSnapshotDir(childName) && isDir && dir.isSnapshottable()) {
// skip the ".snapshot" in components
count++;
index++;
@@ -222,8 +220,7 @@ public class INodesInPath {
break;
}
// Resolve snapshot root
- final Snapshot s = ((INodeDirectorySnapshottable)dir).getSnapshot(
- components[count + 1]);
+ final Snapshot s = dir.getSnapshot(components[count + 1]);
if (s == null) {
//snapshot not found
curNode = null;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Tue Jul 15 21:10:24 2014
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
@@ -47,6 +50,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.RemotePeerFactory;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -55,6 +59,12 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
+import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
+import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
+import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
@@ -65,6 +75,7 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
@@ -92,7 +103,7 @@ import com.google.common.annotations.Vis
* factors of each file.
*/
@InterfaceAudience.Private
-public class NamenodeFsck {
+public class NamenodeFsck implements DataEncryptionKeyFactory {
public static final Log LOG = LogFactory.getLog(NameNode.class.getName());
// return string marking fsck status
@@ -149,6 +160,7 @@ public class NamenodeFsck {
private List<String> snapshottableDirs = null;
private final BlockPlacementPolicy bpPolicy;
+ private final SaslDataTransferClient saslClient;
/**
* Filesystem checker.
@@ -175,6 +187,12 @@ public class NamenodeFsck {
networktopology,
namenode.getNamesystem().getBlockManager().getDatanodeManager()
.getHost2DatanodeMap());
+ this.saslClient = new SaslDataTransferClient(
+ DataTransferSaslUtil.getSaslPropertiesResolver(conf),
+ TrustedChannelResolver.getInstance(conf),
+ conf.getBoolean(
+ IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
+ IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT));
for (Iterator<String> it = pmap.keySet().iterator(); it.hasNext();) {
String key = it.next();
@@ -616,15 +634,16 @@ public class NamenodeFsck {
setConfiguration(namenode.conf).
setRemotePeerFactory(new RemotePeerFactory() {
@Override
- public Peer newConnectedPeer(InetSocketAddress addr)
+ public Peer newConnectedPeer(InetSocketAddress addr,
+ Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
try {
s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
- peer = TcpPeerServer.peerFromSocketAndKey(s, namenode.getRpcServer().
- getDataEncryptionKey());
+ peer = TcpPeerServer.peerFromSocketAndKey(saslClient, s,
+ NamenodeFsck.this, blockToken, datanodeId);
} finally {
if (peer == null) {
IOUtils.closeQuietly(s);
@@ -663,7 +682,12 @@ public class NamenodeFsck {
throw new Exception("Could not copy block data for " + lblock.getBlock());
}
}
-
+
+ @Override
+ public DataEncryptionKey newDataEncryptionKey() throws IOException {
+ return namenode.getRpcServer().getDataEncryptionKey();
+ }
+
/*
* XXX (ab) See comment above for copyBlock().
*
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java Tue Jul 15 21:10:24 2014
@@ -48,7 +48,9 @@ import org.apache.hadoop.hdfs.util.ReadO
import com.google.common.base.Preconditions;
/**
- * Feature for directory with snapshot-related information.
+ * Feature used to store and process the snapshot diff information for a
+ * directory. In particular, it contains a directory diff list recording changes
+ * made to the directory and its children for each snapshot.
*/
@InterfaceAudience.Private
public class DirectoryWithSnapshotFeature implements INode.Feature {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java Tue Jul 15 21:10:24 2014
@@ -127,9 +127,8 @@ public class FSImageFormatPBSnapshot {
}
/**
- * Load the snapshots section from fsimage. Also convert snapshottable
- * directories into {@link INodeDirectorySnapshottable}.
- *
+ * Load the snapshots section from fsimage. Also add snapshottable feature
+ * to snapshottable directories.
*/
public void loadSnapshotSection(InputStream in) throws IOException {
SnapshotManager sm = fsn.getSnapshotManager();
@@ -139,16 +138,13 @@ public class FSImageFormatPBSnapshot {
sm.setSnapshotCounter(section.getSnapshotCounter());
for (long sdirId : section.getSnapshottableDirList()) {
INodeDirectory dir = fsDir.getInode(sdirId).asDirectory();
- final INodeDirectorySnapshottable sdir;
if (!dir.isSnapshottable()) {
- sdir = new INodeDirectorySnapshottable(dir);
- fsDir.addToInodeMap(sdir);
+ dir.addSnapshottableFeature();
} else {
// dir is root, and admin set root to snapshottable before
- sdir = (INodeDirectorySnapshottable) dir;
- sdir.setSnapshotQuota(INodeDirectorySnapshottable.SNAPSHOT_LIMIT);
+ dir.setSnapshotQuota(DirectorySnapshottableFeature.SNAPSHOT_LIMIT);
}
- sm.addSnapshottable(sdir);
+ sm.addSnapshottable(dir);
}
loadSnapshots(in, snum);
}
@@ -160,12 +156,11 @@ public class FSImageFormatPBSnapshot {
INodeDirectory root = loadINodeDirectory(pbs.getRoot(),
parent.getLoaderContext());
int sid = pbs.getSnapshotId();
- INodeDirectorySnapshottable parent = (INodeDirectorySnapshottable) fsDir
- .getInode(root.getId()).asDirectory();
+ INodeDirectory parent = fsDir.getInode(root.getId()).asDirectory();
Snapshot snapshot = new Snapshot(sid, root, parent);
// add the snapshot to parent, since we follow the sequence of
// snapshotsByNames when saving, we do not need to sort when loading
- parent.addSnapshot(snapshot);
+ parent.getDirectorySnapshottableFeature().addSnapshot(snapshot);
snapshotMap.put(sid, snapshot);
}
}
@@ -373,14 +368,15 @@ public class FSImageFormatPBSnapshot {
.setSnapshotCounter(sm.getSnapshotCounter())
.setNumSnapshots(sm.getNumSnapshots());
- INodeDirectorySnapshottable[] snapshottables = sm.getSnapshottableDirs();
- for (INodeDirectorySnapshottable sdir : snapshottables) {
+ INodeDirectory[] snapshottables = sm.getSnapshottableDirs();
+ for (INodeDirectory sdir : snapshottables) {
b.addSnapshottableDir(sdir.getId());
}
b.build().writeDelimitedTo(out);
int i = 0;
- for(INodeDirectorySnapshottable sdir : snapshottables) {
- for(Snapshot s : sdir.getSnapshotsByNames()) {
+ for(INodeDirectory sdir : snapshottables) {
+ for (Snapshot s : sdir.getDirectorySnapshottableFeature()
+ .getSnapshotList()) {
Root sroot = s.getRoot();
SnapshotSection.Snapshot.Builder sb = SnapshotSection.Snapshot
.newBuilder().setSnapshotId(s.getId());
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java Tue Jul 15 21:10:24 2014
@@ -184,15 +184,14 @@ public class Snapshot implements Compara
/** The root directory of the snapshot. */
private final Root root;
- Snapshot(int id, String name, INodeDirectorySnapshottable dir) {
+ Snapshot(int id, String name, INodeDirectory dir) {
this(id, dir, dir);
this.root.setLocalName(DFSUtil.string2Bytes(name));
}
- Snapshot(int id, INodeDirectory dir, INodeDirectorySnapshottable parent) {
+ Snapshot(int id, INodeDirectory dir, INodeDirectory parent) {
this.id = id;
this.root = new Root(dir);
-
this.root.setParent(parent);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java Tue Jul 15 21:10:24 2014
@@ -99,7 +99,7 @@ class SnapshotDiffInfo {
}
/** The root directory of the snapshots */
- private final INodeDirectorySnapshottable snapshotRoot;
+ private final INodeDirectory snapshotRoot;
/** The starting point of the difference */
private final Snapshot from;
/** The end point of the difference */
@@ -122,8 +122,8 @@ class SnapshotDiffInfo {
private final Map<Long, RenameEntry> renameMap =
new HashMap<Long, RenameEntry>();
- SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
- Snapshot end) {
+ SnapshotDiffInfo(INodeDirectory snapshotRoot, Snapshot start, Snapshot end) {
+ Preconditions.checkArgument(snapshotRoot.isSnapshottable());
this.snapshotRoot = snapshotRoot;
this.from = start;
this.to = end;
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java Tue Jul 15 21:10:24 2014
@@ -41,6 +41,8 @@ import org.apache.hadoop.hdfs.tools.snap
import org.apache.hadoop.hdfs.util.Diff.ListType;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import com.google.common.base.Preconditions;
+
/**
* A helper class defining static methods for reading/writing snapshot related
* information from/to FSImage.
@@ -52,17 +54,19 @@ public class SnapshotFSImageFormat {
* @param out The {@link DataOutput} to write.
* @throws IOException
*/
- public static void saveSnapshots(INodeDirectorySnapshottable current,
- DataOutput out) throws IOException {
+ public static void saveSnapshots(INodeDirectory current, DataOutput out)
+ throws IOException {
+ DirectorySnapshottableFeature sf = current.getDirectorySnapshottableFeature();
+ Preconditions.checkArgument(sf != null);
// list of snapshots in snapshotsByNames
- ReadOnlyList<Snapshot> snapshots = current.getSnapshotsByNames();
+ ReadOnlyList<Snapshot> snapshots = sf.getSnapshotList();
out.writeInt(snapshots.size());
for (Snapshot s : snapshots) {
// write the snapshot id
out.writeInt(s.getId());
}
// snapshot quota
- out.writeInt(current.getSnapshotQuota());
+ out.writeInt(sf.getSnapshotQuota());
}
/**
@@ -216,19 +220,22 @@ public class SnapshotFSImageFormat {
* @param loader
* The loader
*/
- public static void loadSnapshotList(
- INodeDirectorySnapshottable snapshottableParent, int numSnapshots,
- DataInput in, FSImageFormat.Loader loader) throws IOException {
+ public static void loadSnapshotList(INodeDirectory snapshottableParent,
+ int numSnapshots, DataInput in, FSImageFormat.Loader loader)
+ throws IOException {
+ DirectorySnapshottableFeature sf = snapshottableParent
+ .getDirectorySnapshottableFeature();
+ Preconditions.checkArgument(sf != null);
for (int i = 0; i < numSnapshots; i++) {
// read snapshots
final Snapshot s = loader.getSnapshot(in);
s.getRoot().setParent(snapshottableParent);
- snapshottableParent.addSnapshot(s);
+ sf.addSnapshot(s);
}
int snapshotQuota = in.readInt();
snapshottableParent.setSnapshotQuota(snapshotQuota);
}
-
+
/**
* Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
* directory.
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Tue Jul 15 21:10:24 2014
@@ -44,6 +44,8 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.metrics2.util.MBeans;
+import com.google.common.base.Preconditions;
+
/**
* Manage snapshottable directories and their snapshots.
*
@@ -66,8 +68,8 @@ public class SnapshotManager implements
private int snapshotCounter = 0;
/** All snapshottable directories in the namesystem. */
- private final Map<Long, INodeDirectorySnapshottable> snapshottables
- = new HashMap<Long, INodeDirectorySnapshottable>();
+ private final Map<Long, INodeDirectory> snapshottables =
+ new HashMap<Long, INodeDirectory>();
public SnapshotManager(final FSDirectory fsdir) {
this.fsdir = fsdir;
@@ -84,7 +86,7 @@ public class SnapshotManager implements
return;
}
- for(INodeDirectorySnapshottable s : snapshottables.values()) {
+ for(INodeDirectory s : snapshottables.values()) {
if (s.isAncestorDirectory(dir)) {
throw new SnapshotException(
"Nested snapshottable directories not allowed: path=" + path
@@ -112,33 +114,30 @@ public class SnapshotManager implements
checkNestedSnapshottable(d, path);
}
-
- final INodeDirectorySnapshottable s;
if (d.isSnapshottable()) {
//The directory is already a snapshottable directory.
- s = (INodeDirectorySnapshottable)d;
- s.setSnapshotQuota(INodeDirectorySnapshottable.SNAPSHOT_LIMIT);
+ d.setSnapshotQuota(DirectorySnapshottableFeature.SNAPSHOT_LIMIT);
} else {
- s = d.replaceSelf4INodeDirectorySnapshottable(iip.getLatestSnapshotId(),
- fsdir.getINodeMap());
+ d.addSnapshottableFeature();
}
- addSnapshottable(s);
+ addSnapshottable(d);
}
/** Add the given snapshottable directory to {@link #snapshottables}. */
- public void addSnapshottable(INodeDirectorySnapshottable dir) {
+ public void addSnapshottable(INodeDirectory dir) {
+ Preconditions.checkArgument(dir.isSnapshottable());
snapshottables.put(dir.getId(), dir);
}
/** Remove the given snapshottable directory from {@link #snapshottables}. */
- private void removeSnapshottable(INodeDirectorySnapshottable s) {
+ private void removeSnapshottable(INodeDirectory s) {
snapshottables.remove(s.getId());
}
/** Remove snapshottable directories from {@link #snapshottables} */
- public void removeSnapshottable(List<INodeDirectorySnapshottable> toRemove) {
+ public void removeSnapshottable(List<INodeDirectory> toRemove) {
if (toRemove != null) {
- for (INodeDirectorySnapshottable s : toRemove) {
+ for (INodeDirectory s : toRemove) {
removeSnapshottable(s);
}
}
@@ -152,22 +151,22 @@ public class SnapshotManager implements
public void resetSnapshottable(final String path) throws IOException {
final INodesInPath iip = fsdir.getINodesInPath4Write(path);
final INodeDirectory d = INodeDirectory.valueOf(iip.getLastINode(), path);
- if (!d.isSnapshottable()) {
+ DirectorySnapshottableFeature sf = d.getDirectorySnapshottableFeature();
+ if (sf == null) {
// the directory is already non-snapshottable
return;
}
- final INodeDirectorySnapshottable s = (INodeDirectorySnapshottable) d;
- if (s.getNumSnapshots() > 0) {
+ if (sf.getNumSnapshots() > 0) {
throw new SnapshotException("The directory " + path + " has snapshot(s). "
+ "Please redo the operation after removing all the snapshots.");
}
- if (s == fsdir.getRoot()) {
- s.setSnapshotQuota(0);
+ if (d == fsdir.getRoot()) {
+ d.setSnapshotQuota(0);
} else {
- s.replaceSelf(iip.getLatestSnapshotId(), fsdir.getINodeMap());
+ d.removeSnapshottableFeature();
}
- removeSnapshottable(s);
+ removeSnapshottable(d);
}
/**
@@ -180,10 +179,15 @@ public class SnapshotManager implements
* Throw IOException when the given path does not lead to an
* existing snapshottable directory.
*/
- public INodeDirectorySnapshottable getSnapshottableRoot(final String path
- ) throws IOException {
- final INodesInPath i = fsdir.getINodesInPath4Write(path);
- return INodeDirectorySnapshottable.valueOf(i.getLastINode(), path);
+ public INodeDirectory getSnapshottableRoot(final String path)
+ throws IOException {
+ final INodeDirectory dir = INodeDirectory.valueOf(fsdir
+ .getINodesInPath4Write(path).getLastINode(), path);
+ if (!dir.isSnapshottable()) {
+ throw new SnapshotException(
+ "Directory is not a snapshottable directory: " + path);
+ }
+ return dir;
}
/**
@@ -202,7 +206,7 @@ public class SnapshotManager implements
*/
public String createSnapshot(final String path, String snapshotName
) throws IOException {
- INodeDirectorySnapshottable srcRoot = getSnapshottableRoot(path);
+ INodeDirectory srcRoot = getSnapshottableRoot(path);
if (snapshotCounter == getMaxSnapshotID()) {
// We have reached the maximum allowable snapshot ID and since we don't
@@ -235,7 +239,7 @@ public class SnapshotManager implements
// parse the path, and check if the path is a snapshot path
// the INodeDirectorySnapshottable#valueOf method will throw Exception
// if the path is not for a snapshottable directory
- INodeDirectorySnapshottable srcRoot = getSnapshottableRoot(path);
+ INodeDirectory srcRoot = getSnapshottableRoot(path);
srcRoot.removeSnapshot(snapshotName, collectedBlocks, removedINodes);
numSnapshots.getAndDecrement();
}
@@ -258,8 +262,7 @@ public class SnapshotManager implements
final String newSnapshotName) throws IOException {
// Find the source root directory path where the snapshot was taken.
// All the check for path has been included in the valueOf method.
- final INodeDirectorySnapshottable srcRoot
- = INodeDirectorySnapshottable.valueOf(fsdir.getINode(path), path);
+ final INodeDirectory srcRoot = getSnapshottableRoot(path);
// Note that renameSnapshot and createSnapshot are synchronized externally
// through FSNamesystem's write lock
srcRoot.renameSnapshot(path, oldSnapshotName, newSnapshotName);
@@ -285,9 +288,9 @@ public class SnapshotManager implements
snapshotCounter = counter;
}
- INodeDirectorySnapshottable[] getSnapshottableDirs() {
+ INodeDirectory[] getSnapshottableDirs() {
return snapshottables.values().toArray(
- new INodeDirectorySnapshottable[snapshottables.size()]);
+ new INodeDirectory[snapshottables.size()]);
}
/**
@@ -299,8 +302,9 @@ public class SnapshotManager implements
out.writeInt(numSnapshots.get());
// write all snapshots.
- for(INodeDirectorySnapshottable snapshottableDir : snapshottables.values()) {
- for(Snapshot s : snapshottableDir.getSnapshotsByNames()) {
+ for(INodeDirectory snapshottableDir : snapshottables.values()) {
+ for (Snapshot s : snapshottableDir.getDirectorySnapshottableFeature()
+ .getSnapshotList()) {
s.write(out);
}
}
@@ -339,16 +343,16 @@ public class SnapshotManager implements
List<SnapshottableDirectoryStatus> statusList =
new ArrayList<SnapshottableDirectoryStatus>();
- for (INodeDirectorySnapshottable dir : snapshottables.values()) {
+ for (INodeDirectory dir : snapshottables.values()) {
if (userName == null || userName.equals(dir.getUserName())) {
SnapshottableDirectoryStatus status = new SnapshottableDirectoryStatus(
dir.getModificationTime(), dir.getAccessTime(),
dir.getFsPermission(), dir.getUserName(), dir.getGroupName(),
dir.getLocalNameBytes(), dir.getId(),
dir.getChildrenNum(Snapshot.CURRENT_STATE_ID),
- dir.getNumSnapshots(),
- dir.getSnapshotQuota(), dir.getParent() == null ?
- DFSUtil.EMPTY_BYTES :
+ dir.getDirectorySnapshottableFeature().getNumSnapshots(),
+ dir.getDirectorySnapshottableFeature().getSnapshotQuota(),
+ dir.getParent() == null ? DFSUtil.EMPTY_BYTES :
DFSUtil.string2Bytes(dir.getParent().getFullPathName()));
statusList.add(status);
}
@@ -364,20 +368,18 @@ public class SnapshotManager implements
*/
public SnapshotDiffReport diff(final String path, final String from,
final String to) throws IOException {
+ // Find the source root directory path where the snapshots were taken.
+ // All the check for path has been included in the valueOf method.
+ final INodeDirectory snapshotRoot = getSnapshottableRoot(path);
+
if ((from == null || from.isEmpty())
&& (to == null || to.isEmpty())) {
// both fromSnapshot and toSnapshot indicate the current tree
return new SnapshotDiffReport(path, from, to,
Collections.<DiffReportEntry> emptyList());
}
-
- // Find the source root directory path where the snapshots were taken.
- // All the check for path has been included in the valueOf method.
- INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString());
- final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
- .valueOf(inodesInPath.getLastINode(), path);
-
- final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to);
+ final SnapshotDiffInfo diffs = snapshotRoot
+ .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to);
return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
path, from, to, Collections.<DiffReportEntry> emptyList());
}
@@ -412,7 +414,7 @@ public class SnapshotManager implements
getSnapshottableDirectories() {
List<SnapshottableDirectoryStatus.Bean> beans =
new ArrayList<SnapshottableDirectoryStatus.Bean>();
- for (INodeDirectorySnapshottable d : getSnapshottableDirs()) {
+ for (INodeDirectory d : getSnapshottableDirs()) {
beans.add(toBean(d));
}
return beans.toArray(new SnapshottableDirectoryStatus.Bean[beans.size()]);
@@ -421,20 +423,19 @@ public class SnapshotManager implements
@Override // SnapshotStatsMXBean
public SnapshotInfo.Bean[] getSnapshots() {
List<SnapshotInfo.Bean> beans = new ArrayList<SnapshotInfo.Bean>();
- for (INodeDirectorySnapshottable d : getSnapshottableDirs()) {
- for (Snapshot s : d.getSnapshotList()) {
+ for (INodeDirectory d : getSnapshottableDirs()) {
+ for (Snapshot s : d.getDirectorySnapshottableFeature().getSnapshotList()) {
beans.add(toBean(s));
}
}
return beans.toArray(new SnapshotInfo.Bean[beans.size()]);
}
- public static SnapshottableDirectoryStatus.Bean toBean(
- INodeDirectorySnapshottable d) {
+ public static SnapshottableDirectoryStatus.Bean toBean(INodeDirectory d) {
return new SnapshottableDirectoryStatus.Bean(
d.getFullPathName(),
- d.getNumSnapshots(),
- d.getSnapshotQuota(),
+ d.getDirectorySnapshottableFeature().getNumSnapshots(),
+ d.getDirectorySnapshottableFeature().getSnapshotQuota(),
d.getModificationTime(),
Short.valueOf(Integer.toOctalString(
d.getFsPermissionShort())),
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Tue Jul 15 21:10:24 2014
@@ -1440,6 +1440,12 @@ public class DFSAdmin extends FsShell {
} else if ("-fetchImage".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-fetchImage <local directory>]");
+ } else if ("-shutdownDatanode".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-shutdownDatanode <datanode_host:ipc_port> [upgrade]]");
+ } else if ("-getDatanodeInfo".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-getDatanodeInfo <datanode_host:ipc_port>]");
} else {
System.err.println("Usage: java DFSAdmin");
System.err.println("Note: Administrative commands can only be run as the HDFS superuser.");
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Tue Jul 15 21:10:24 2014
@@ -1452,6 +1452,37 @@
</property>
<property>
+ <name>dfs.data.transfer.protection</name>
+ <value></value>
+ <description>
+ A comma-separated list of SASL protection values used for secured
+ connections to the DataNode when reading or writing block data. Possible
+ values are authentication, integrity and privacy. authentication means
+ authentication only and no integrity or privacy; integrity implies
+ authentication and integrity are enabled; and privacy implies all of
+ authentication, integrity and privacy are enabled. If
+ dfs.encrypt.data.transfer is set to true, then it supersedes the setting for
+ dfs.data.transfer.protection and enforces that all connections must use a
+ specialized encrypted SASL handshake. This property is ignored for
+ connections to a DataNode listening on a privileged port. In this case, it
+ is assumed that the use of a privileged port establishes sufficient trust.
+ </description>
+</property>
+
+<property>
+ <name>dfs.data.transfer.saslproperties.resolver.class</name>
+ <value></value>
+ <description>
+ SaslPropertiesResolver used to resolve the QOP used for a connection to the
+ DataNode when reading or writing block data. If not specified, the full set
+ of values specified in dfs.data.transfer.protection is used while
+ determining the QOP used for the connection. If a class is specified, then
+ the QOP values returned by the class will be used while determining the QOP
+ used for the connection.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>false</value>
<description>
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1608601-1610850
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js Tue Jul 15 21:10:24 2014
@@ -103,7 +103,7 @@
}
var url = '/webhdfs/v1' + abs_path + '?op=GET_BLOCK_LOCATIONS';
- $.ajax({"url": url, "crossDomain": true}).done(function(data) {
+ $.get(url).done(function(data) {
var d = get_response(data, "LocatedBlocks");
if (d === null) {
show_err_msg(get_response_err_msg(data));
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm Tue Jul 15 21:10:24 2014
@@ -270,7 +270,7 @@ Centralized Cache Management in HDFS
** {Native Libraries}
In order to lock block files into memory, the DataNode relies on native JNI
- code found in <<<libhadoop.so>>>. Be sure to
+ code found in <<<libhadoop.so>>> or <<<hadoop.dll>>> on Windows. Be sure to
{{{../hadoop-common/NativeLibraries.html}enable JNI}} if you are using HDFS
centralized cache management.
@@ -283,11 +283,11 @@ Centralized Cache Management in HDFS
* dfs.datanode.max.locked.memory
This determines the maximum amount of memory a DataNode will use for caching.
- The "locked-in-memory size" ulimit (<<<ulimit -l>>>) of the DataNode user
- also needs to be increased to match this parameter (see below section on
- {{OS Limits}}). When setting this value, please remember that you will need
- space in memory for other things as well, such as the DataNode and
- application JVM heaps and the operating system page cache.
+ On Unix-like systems, the "locked-in-memory size" ulimit (<<<ulimit -l>>>) of
+ the DataNode user also needs to be increased to match this parameter (see
+ below section on {{OS Limits}}). When setting this value, please remember
+ that you will need space in memory for other things as well, such as the
+ DataNode and application JVM heaps and the operating system page cache.
*** Optional
@@ -339,3 +339,6 @@ Centralized Cache Management in HDFS
"unlimited," indicating that there is no limit. Note that it's typical for
<<<ulimit -l>>> to output the memory lock limit in KB, but
dfs.datanode.max.locked.memory must be specified in bytes.
+
+ This information does not apply to deployments on Windows. Windows has no
+ direct equivalent of <<<ulimit -l>>>.
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm Tue Jul 15 21:10:24 2014
@@ -838,41 +838,22 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6O
$ mvn clean package -Pdist
- This will generate a jar with the BookKeeperJournalManager, all the dependencies
- needed by the journal manager,
+ This will generate a jar with the BookKeeperJournalManager,
hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-<VERSION>.jar
- Note that the -Pdist part of the build command is important, as otherwise
- the dependencies would not be packaged in the jar. The dependencies included in
- the jar are {{{http://maven.apache.org/plugins/maven-shade-plugin/}shaded}} to
- avoid conflicts with other dependencies of the NameNode.
+ Note that the -Pdist part of the build command is important, this would
+ copy the dependent bookkeeper-server jar under
+ hadoop-hdfs/src/contrib/bkjournal/target/lib.
*** <<Putting the BookKeeperJournalManager in the NameNode classpath>>
- To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal
- jar, generated above, into the lib directory of hdfs. In the standard
- distribution of HDFS, this is at $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
+ To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal and
+ bookkeeper-server jar, mentioned above, into the lib directory of hdfs. In the
+ standard distribution of HDFS, this is at $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
cp hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-<VERSION>.jar $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
*** <<Current limitations>>
- 1) NameNode format command will not format the BookKeeper data automatically.
- We have to clean the data manually from BookKeeper cluster
- and create the /ledgers/available path in Zookeeper.
-----
-$ zkCli.sh create /ledgers 0
-$ zkCli.sh create /ledgers/available 0
-----
- Note:
- bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal
- The final part /hdfsjournal specifies the znode in zookeeper where
- ledger metadata will be stored. Administrators may set this to anything
- they wish.
-
- 2) Security in BookKeeper. BookKeeper does not support SASL nor SSL for
- connections between the NameNode and BookKeeper storage nodes.
-
- 3) Auto-Recovery of storage node failures. Work inprogress
- {{{https://issues.apache.org/jira/browse/BOOKKEEPER-237 }BOOKKEEPER-237}}.
- Currently we have the tools to manually recover the data from failed storage nodes.
+ 1) Security in BookKeeper. BookKeeper does not support SASL nor SSL for
+ connections between the NameNode and BookKeeper storage nodes.
\ No newline at end of file
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm Tue Jul 15 21:10:24 2014
@@ -394,7 +394,7 @@ Hello, webhdfs user!
* Submit a HTTP PUT request.
+---------------------------------
-curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
+curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
+---------------------------------
The client receives a response with a {{{Boolean JSON Schema}<<<boolean>>> JSON object}}:
@@ -419,7 +419,7 @@ Transfer-Encoding: chunked
* Submit a HTTP PUT request.
+---------------------------------
-curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=CREATESYMLINK
+curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATESYMLINK
&destination=<PATH>[&createParent=<true|false>]"
+---------------------------------
@@ -934,7 +934,7 @@ Transfer-Encoding: chunked
* Submit a HTTP PUT request.
+---------------------------------
-curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=op=SETXATTR
+curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETXATTR
&xattr.name=<XATTRNAME>&xattr.value=<XATTRVALUE>
&flag=<FLAG>"
+---------------------------------
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java?rev=1610853&r1=1610852&r2=1610853&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java Tue Jul 15 21:10:24 2014
@@ -33,9 +33,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
@@ -48,6 +50,7 @@ import org.apache.hadoop.hdfs.shortcircu
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -192,7 +195,8 @@ public class BlockReaderTestUtil {
setAllowShortCircuitLocalReads(true).
setRemotePeerFactory(new RemotePeerFactory() {
@Override
- public Peer newConnectedPeer(InetSocketAddress addr)
+ public Peer newConnectedPeer(InetSocketAddress addr,
+ Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket sock = NetUtils.
@@ -251,4 +255,4 @@ public class BlockReaderTestUtil {
LogManager.getLogger(DataNode.class.getName()).setLevel(
Level.TRACE);
}
-}
\ No newline at end of file
+}