You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2013/12/19 03:04:10 UTC
svn commit: r1552205 [3/6] - in
/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs/
hadoop-hdfs/dev-support/ hadoop-hdfs/src/main/java/
hadoop-hdfs/src/main/java/org/apache/...
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Dec 19 02:03:47 2013
@@ -67,7 +67,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
import org.apache.hadoop.hdfs.util.ByteArray;
@@ -622,8 +621,7 @@ public class FSDirectory implements Clos
// snapshot is taken on the dst tree, changes will be recorded in the latest
// snapshot of the src tree.
if (isSrcInSnapshot) {
- srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot(),
- inodeMap);
+ srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot());
srcIIP.setLastINode(srcChild);
}
@@ -692,11 +690,9 @@ public class FSDirectory implements Clos
}
// update modification time of dst and the parent of src
final INode srcParent = srcIIP.getINode(-2);
- srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot(),
- inodeMap);
+ srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot());
dstParent = dstIIP.getINode(-2); // refresh dstParent
- dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot(),
- inodeMap);
+ dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot());
// update moved leases with new filename
getFSNamesystem().unprotectedChangeLease(src, dst);
@@ -734,11 +730,10 @@ public class FSDirectory implements Clos
}
if (isSrcInSnapshot) {
- // srcParent must be an INodeDirectoryWithSnapshot instance since
- // isSrcInSnapshot is true and src node has been removed from
- // srcParent
- ((INodeDirectoryWithSnapshot) srcParent).undoRename4ScrParent(
- oldSrcChild.asReference(), srcChild, srcIIP.getLatestSnapshot());
+ // srcParent must have snapshot feature since isSrcInSnapshot is true
+ // and src node has been removed from srcParent
+ srcParent.undoRename4ScrParent(oldSrcChild.asReference(), srcChild,
+ srcIIP.getLatestSnapshot());
} else {
// original srcChild is not in latest snapshot, we only need to add
// the srcChild back
@@ -879,8 +874,7 @@ public class FSDirectory implements Clos
// snapshot is taken on the dst tree, changes will be recorded in the latest
// snapshot of the src tree.
if (isSrcInSnapshot) {
- srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot(),
- inodeMap);
+ srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot());
srcIIP.setLastINode(srcChild);
}
@@ -958,11 +952,9 @@ public class FSDirectory implements Clos
}
final INode srcParent = srcIIP.getINode(-2);
- srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot(),
- inodeMap);
+ srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot());
dstParent = dstIIP.getINode(-2);
- dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot(),
- inodeMap);
+ dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot());
// update moved lease with new filename
getFSNamesystem().unprotectedChangeLease(src, dst);
@@ -1019,9 +1011,9 @@ public class FSDirectory implements Clos
withCount.getReferredINode().setLocalName(srcChildName);
}
- if (srcParent instanceof INodeDirectoryWithSnapshot) {
- ((INodeDirectoryWithSnapshot) srcParent).undoRename4ScrParent(
- oldSrcChild.asReference(), srcChild, srcIIP.getLatestSnapshot());
+ if (srcParent.isWithSnapshot()) {
+ srcParent.undoRename4ScrParent(oldSrcChild.asReference(), srcChild,
+ srcIIP.getLatestSnapshot());
} else {
// srcParent is not an INodeDirectoryWithSnapshot, we only need to add
// the srcChild back
@@ -1030,9 +1022,9 @@ public class FSDirectory implements Clos
}
if (undoRemoveDst) {
// Rename failed - restore dst
- if (dstParent instanceof INodeDirectoryWithSnapshot) {
- ((INodeDirectoryWithSnapshot) dstParent).undoRename4DstParent(
- removedDst, dstIIP.getLatestSnapshot());
+ if (dstParent.isDirectory() && dstParent.asDirectory().isWithSnapshot()) {
+ dstParent.asDirectory().undoRename4DstParent(removedDst,
+ dstIIP.getLatestSnapshot());
} else {
addLastINodeNoQuotaCheck(dstIIP, removedDst);
}
@@ -1163,8 +1155,7 @@ public class FSDirectory implements Clos
if (inode == null) {
throw new FileNotFoundException("File does not exist: " + src);
}
- inode.setPermission(permissions, inodesInPath.getLatestSnapshot(),
- inodeMap);
+ inode.setPermission(permissions, inodesInPath.getLatestSnapshot());
}
void setOwner(String src, String username, String groupname)
@@ -1189,11 +1180,10 @@ public class FSDirectory implements Clos
throw new FileNotFoundException("File does not exist: " + src);
}
if (username != null) {
- inode = inode.setUser(username, inodesInPath.getLatestSnapshot(),
- inodeMap);
+ inode = inode.setUser(username, inodesInPath.getLatestSnapshot());
}
if (groupname != null) {
- inode.setGroup(groupname, inodesInPath.getLatestSnapshot(), inodeMap);
+ inode.setGroup(groupname, inodesInPath.getLatestSnapshot());
}
}
@@ -1266,7 +1256,7 @@ public class FSDirectory implements Clos
if(nodeToRemove == null) continue;
nodeToRemove.setBlocks(null);
- trgParent.removeChild(nodeToRemove, trgLatestSnapshot, null);
+ trgParent.removeChild(nodeToRemove, trgLatestSnapshot);
inodeMap.remove(nodeToRemove);
count++;
}
@@ -1274,8 +1264,8 @@ public class FSDirectory implements Clos
// update inodeMap
removeFromInodeMap(Arrays.asList(allSrcInodes));
- trgInode.setModificationTime(timestamp, trgLatestSnapshot, inodeMap);
- trgParent.updateModificationTime(timestamp, trgLatestSnapshot, inodeMap);
+ trgInode.setModificationTime(timestamp, trgLatestSnapshot);
+ trgParent.updateModificationTime(timestamp, trgLatestSnapshot);
// update quota on the parent directory ('count' files removed, 0 space)
unprotectedUpdateCount(trgIIP, trgINodes.length-1, -count, 0);
}
@@ -1419,7 +1409,7 @@ public class FSDirectory implements Clos
// record modification
final Snapshot latestSnapshot = iip.getLatestSnapshot();
- targetNode = targetNode.recordModification(latestSnapshot, inodeMap);
+ targetNode = targetNode.recordModification(latestSnapshot);
iip.setLastINode(targetNode);
// Remove the node from the namespace
@@ -1430,7 +1420,7 @@ public class FSDirectory implements Clos
// set the parent's modification time
final INodeDirectory parent = targetNode.getParent();
- parent.updateModificationTime(mtime, latestSnapshot, inodeMap);
+ parent.updateModificationTime(mtime, latestSnapshot);
if (removed == 0) {
return 0;
}
@@ -2203,8 +2193,7 @@ public class FSDirectory implements Clos
final INodeDirectory parent = inodes[pos-1].asDirectory();
boolean added = false;
try {
- added = parent.addChild(child, true, iip.getLatestSnapshot(),
- inodeMap);
+ added = parent.addChild(child, true, iip.getLatestSnapshot());
} catch (QuotaExceededException e) {
updateCountNoQuotaCheck(iip, pos,
-counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
@@ -2242,7 +2231,7 @@ public class FSDirectory implements Clos
final Snapshot latestSnapshot = iip.getLatestSnapshot();
final INode last = iip.getLastINode();
final INodeDirectory parent = iip.getINode(-2).asDirectory();
- if (!parent.removeChild(last, latestSnapshot, inodeMap)) {
+ if (!parent.removeChild(last, latestSnapshot)) {
return -1;
}
INodeDirectory newParent = last.getParent();
@@ -2394,7 +2383,7 @@ public class FSDirectory implements Clos
}
final Snapshot latest = iip.getLatestSnapshot();
- dirNode = dirNode.recordModification(latest, inodeMap);
+ dirNode = dirNode.recordModification(latest);
dirNode.setQuota(nsQuota, dsQuota);
return dirNode;
}
@@ -2462,7 +2451,7 @@ public class FSDirectory implements Clos
assert hasWriteLock();
boolean status = false;
if (mtime != -1) {
- inode = inode.setModificationTime(mtime, latest, inodeMap);
+ inode = inode.setModificationTime(mtime, latest);
status = true;
}
if (atime != -1) {
@@ -2473,7 +2462,7 @@ public class FSDirectory implements Clos
if (atime <= inodeTime + getFSNamesystem().getAccessTimePrecision() && !force) {
status = false;
} else {
- inode.setAccessTime(atime, latest, inodeMap);
+ inode.setAccessTime(atime, latest);
status = true;
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Thu Dec 19 02:03:47 2013
@@ -160,10 +160,10 @@ public class FSEditLog implements LogsPu
private long totalTimeTransactions; // total time for all transactions
private NameNodeMetrics metrics;
- private NNStorage storage;
- private Configuration conf;
+ private final NNStorage storage;
+ private final Configuration conf;
- private List<URI> editsDirs;
+ private final List<URI> editsDirs;
private ThreadLocal<OpInstanceCache> cache =
new ThreadLocal<OpInstanceCache>() {
@@ -176,7 +176,7 @@ public class FSEditLog implements LogsPu
/**
* The edit directories that are shared between primary and secondary.
*/
- private List<URI> sharedEditsDirs;
+ private final List<URI> sharedEditsDirs;
private static class TransactionId {
public long txid;
@@ -203,10 +203,6 @@ public class FSEditLog implements LogsPu
* @param editsDirs List of journals to use
*/
FSEditLog(Configuration conf, NNStorage storage, List<URI> editsDirs) {
- init(conf, storage, editsDirs);
- }
-
- private void init(Configuration conf, NNStorage storage, List<URI> editsDirs) {
isSyncRunning = false;
this.conf = conf;
this.storage = storage;
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Thu Dec 19 02:03:47 2013
@@ -24,25 +24,27 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.EnumMap;
+import java.util.EnumSet;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCacheDirectiveInfoOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCachePoolOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCloseOp;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCacheDirectiveInfoOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AllocateBlockIdOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AllowSnapshotOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.BlockListUpdatingOp;
@@ -55,11 +57,11 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.DisallowSnapshotOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.GetDelegationTokenOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ModifyCachePoolOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ModifyCacheDirectiveInfoOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ModifyCachePoolOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ReassignLeaseOp;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveCachePoolOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveCacheDirectiveInfoOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveCachePoolOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOldOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameSnapshotOp;
@@ -354,8 +356,8 @@ public class FSEditLogLoader {
// update the block list.
// Update the salient file attributes.
- newFile.setAccessTime(addCloseOp.atime, null, fsDir.getINodeMap());
- newFile.setModificationTime(addCloseOp.mtime, null, fsDir.getINodeMap());
+ newFile.setAccessTime(addCloseOp.atime, null);
+ newFile.setModificationTime(addCloseOp.mtime, null);
updateBlocks(fsDir, addCloseOp, newFile);
break;
}
@@ -373,8 +375,8 @@ public class FSEditLogLoader {
final INodeFile file = INodeFile.valueOf(iip.getINode(0), addCloseOp.path);
// Update the salient file attributes.
- file.setAccessTime(addCloseOp.atime, null, fsDir.getINodeMap());
- file.setModificationTime(addCloseOp.mtime, null, fsDir.getINodeMap());
+ file.setAccessTime(addCloseOp.atime, null);
+ file.setModificationTime(addCloseOp.mtime, null);
updateBlocks(fsDir, addCloseOp, file);
// Now close the file
@@ -650,7 +652,7 @@ public class FSEditLogLoader {
ModifyCacheDirectiveInfoOp modifyOp =
(ModifyCacheDirectiveInfoOp) op;
fsNamesys.getCacheManager().modifyDirective(
- modifyOp.directive, null);
+ modifyOp.directive, null, EnumSet.of(CacheFlag.FORCE));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java Thu Dec 19 02:03:47 2013
@@ -64,7 +64,6 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Date;
import java.util.EnumMap;
import java.util.List;
import java.util.zip.CheckedInputStream;
@@ -76,7 +75,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.Options.Rename;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -120,11 +118,10 @@ import com.google.common.base.Preconditi
@InterfaceStability.Unstable
public abstract class FSEditLogOp {
public final FSEditLogOpCodes opCode;
- long txid;
+ long txid = HdfsConstants.INVALID_TXID;
byte[] rpcClientId = RpcConstants.DUMMY_CLIENT_ID;
int rpcCallId = RpcConstants.INVALID_CALL_ID;
- @SuppressWarnings("deprecation")
final public static class OpInstanceCache {
private EnumMap<FSEditLogOpCodes, FSEditLogOp> inst =
new EnumMap<FSEditLogOpCodes, FSEditLogOp>(FSEditLogOpCodes.class);
@@ -149,13 +146,10 @@ public abstract class FSEditLogOp {
inst.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
inst.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
inst.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
- inst.put(OP_CANCEL_DELEGATION_TOKEN,
- new CancelDelegationTokenOp());
+ inst.put(OP_CANCEL_DELEGATION_TOKEN, new CancelDelegationTokenOp());
inst.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
- inst.put(OP_START_LOG_SEGMENT,
- new LogSegmentOp(OP_START_LOG_SEGMENT));
- inst.put(OP_END_LOG_SEGMENT,
- new LogSegmentOp(OP_END_LOG_SEGMENT));
+ inst.put(OP_START_LOG_SEGMENT, new LogSegmentOp(OP_START_LOG_SEGMENT));
+ inst.put(OP_END_LOG_SEGMENT, new LogSegmentOp(OP_END_LOG_SEGMENT));
inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
inst.put(OP_ALLOW_SNAPSHOT, new AllowSnapshotOp());
@@ -165,12 +159,10 @@ public abstract class FSEditLogOp {
inst.put(OP_RENAME_SNAPSHOT, new RenameSnapshotOp());
inst.put(OP_SET_GENSTAMP_V2, new SetGenstampV2Op());
inst.put(OP_ALLOCATE_BLOCK_ID, new AllocateBlockIdOp());
- inst.put(OP_ADD_CACHE_DIRECTIVE,
- new AddCacheDirectiveInfoOp());
- inst.put(OP_MODIFY_CACHE_DIRECTIVE,
- new ModifyCacheDirectiveInfoOp());
- inst.put(OP_REMOVE_CACHE_DIRECTIVE,
- new RemoveCacheDirectiveInfoOp());
+
+ inst.put(OP_ADD_CACHE_DIRECTIVE, new AddCacheDirectiveInfoOp());
+ inst.put(OP_MODIFY_CACHE_DIRECTIVE, new ModifyCacheDirectiveInfoOp());
+ inst.put(OP_REMOVE_CACHE_DIRECTIVE, new RemoveCacheDirectiveInfoOp());
inst.put(OP_ADD_CACHE_POOL, new AddCachePoolOp());
inst.put(OP_MODIFY_CACHE_POOL, new ModifyCachePoolOp());
inst.put(OP_REMOVE_CACHE_POOL, new RemoveCachePoolOp());
@@ -187,7 +179,6 @@ public abstract class FSEditLogOp {
*/
private FSEditLogOp(FSEditLogOpCodes opCode) {
this.opCode = opCode;
- this.txid = HdfsConstants.INVALID_TXID;
}
public long getTransactionId() {
@@ -2895,56 +2886,25 @@ public abstract class FSEditLogOp {
@Override
void readFields(DataInputStream in, int logVersion) throws IOException {
- long id = FSImageSerialization.readLong(in);
- String path = FSImageSerialization.readString(in);
- short replication = FSImageSerialization.readShort(in);
- String pool = FSImageSerialization.readString(in);
- long expiryTime = FSImageSerialization.readLong(in);
- directive = new CacheDirectiveInfo.Builder().
- setId(id).
- setPath(new Path(path)).
- setReplication(replication).
- setPool(pool).
- setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(expiryTime)).
- build();
+ directive = FSImageSerialization.readCacheDirectiveInfo(in);
readRpcIds(in, logVersion);
}
@Override
public void writeFields(DataOutputStream out) throws IOException {
- FSImageSerialization.writeLong(directive.getId(), out);
- FSImageSerialization.writeString(directive.getPath().toUri().getPath(), out);
- FSImageSerialization.writeShort(directive.getReplication(), out);
- FSImageSerialization.writeString(directive.getPool(), out);
- FSImageSerialization.writeLong(
- directive.getExpiration().getMillis(), out);
+ FSImageSerialization.writeCacheDirectiveInfo(out, directive);
writeRpcIds(rpcClientId, rpcCallId, out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
- XMLUtils.addSaxString(contentHandler, "ID",
- directive.getId().toString());
- XMLUtils.addSaxString(contentHandler, "PATH",
- directive.getPath().toUri().getPath());
- XMLUtils.addSaxString(contentHandler, "REPLICATION",
- Short.toString(directive.getReplication()));
- XMLUtils.addSaxString(contentHandler, "POOL", directive.getPool());
- XMLUtils.addSaxString(contentHandler, "EXPIRATION",
- "" + directive.getExpiration().getMillis());
+ FSImageSerialization.writeCacheDirectiveInfo(contentHandler, directive);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
- directive = new CacheDirectiveInfo.Builder().
- setId(Long.parseLong(st.getValue("ID"))).
- setPath(new Path(st.getValue("PATH"))).
- setReplication(Short.parseShort(st.getValue("REPLICATION"))).
- setPool(st.getValue("POOL")).
- setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(
- Long.parseLong(st.getValue("EXPIRATION")))).
- build();
+ directive = FSImageSerialization.readCacheDirectiveInfo(st);
readRpcIdsFromXml(st);
}
@@ -2988,104 +2948,25 @@ public abstract class FSEditLogOp {
@Override
void readFields(DataInputStream in, int logVersion) throws IOException {
- CacheDirectiveInfo.Builder builder =
- new CacheDirectiveInfo.Builder();
- builder.setId(FSImageSerialization.readLong(in));
- byte flags = in.readByte();
- if ((flags & 0x1) != 0) {
- builder.setPath(new Path(FSImageSerialization.readString(in)));
- }
- if ((flags & 0x2) != 0) {
- builder.setReplication(FSImageSerialization.readShort(in));
- }
- if ((flags & 0x4) != 0) {
- builder.setPool(FSImageSerialization.readString(in));
- }
- if ((flags & 0x8) != 0) {
- builder.setExpiration(
- CacheDirectiveInfo.Expiration.newAbsolute(
- FSImageSerialization.readLong(in)));
- }
- if ((flags & ~0xF) != 0) {
- throw new IOException("unknown flags set in " +
- "ModifyCacheDirectiveInfoOp: " + flags);
- }
- this.directive = builder.build();
+ this.directive = FSImageSerialization.readCacheDirectiveInfo(in);
readRpcIds(in, logVersion);
}
@Override
public void writeFields(DataOutputStream out) throws IOException {
- FSImageSerialization.writeLong(directive.getId(), out);
- byte flags = (byte)(
- ((directive.getPath() != null) ? 0x1 : 0) |
- ((directive.getReplication() != null) ? 0x2 : 0) |
- ((directive.getPool() != null) ? 0x4 : 0) |
- ((directive.getExpiration() != null) ? 0x8 : 0)
- );
- out.writeByte(flags);
- if (directive.getPath() != null) {
- FSImageSerialization.writeString(
- directive.getPath().toUri().getPath(), out);
- }
- if (directive.getReplication() != null) {
- FSImageSerialization.writeShort(directive.getReplication(), out);
- }
- if (directive.getPool() != null) {
- FSImageSerialization.writeString(directive.getPool(), out);
- }
- if (directive.getExpiration() != null) {
- FSImageSerialization.writeLong(directive.getExpiration().getMillis(),
- out);
- }
+ FSImageSerialization.writeCacheDirectiveInfo(out, directive);
writeRpcIds(rpcClientId, rpcCallId, out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
- XMLUtils.addSaxString(contentHandler, "ID",
- Long.toString(directive.getId()));
- if (directive.getPath() != null) {
- XMLUtils.addSaxString(contentHandler, "PATH",
- directive.getPath().toUri().getPath());
- }
- if (directive.getReplication() != null) {
- XMLUtils.addSaxString(contentHandler, "REPLICATION",
- Short.toString(directive.getReplication()));
- }
- if (directive.getPool() != null) {
- XMLUtils.addSaxString(contentHandler, "POOL", directive.getPool());
- }
- if (directive.getExpiration() != null) {
- XMLUtils.addSaxString(contentHandler, "EXPIRATION",
- "" + directive.getExpiration().getMillis());
- }
+ FSImageSerialization.writeCacheDirectiveInfo(contentHandler, directive);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
- CacheDirectiveInfo.Builder builder =
- new CacheDirectiveInfo.Builder();
- builder.setId(Long.parseLong(st.getValue("ID")));
- String path = st.getValueOrNull("PATH");
- if (path != null) {
- builder.setPath(new Path(path));
- }
- String replicationString = st.getValueOrNull("REPLICATION");
- if (replicationString != null) {
- builder.setReplication(Short.parseShort(replicationString));
- }
- String pool = st.getValueOrNull("POOL");
- if (pool != null) {
- builder.setPool(pool);
- }
- String expiryTime = st.getValueOrNull("EXPIRATION");
- if (expiryTime != null) {
- builder.setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(
- Long.parseLong(expiryTime)));
- }
- this.directive = builder.build();
+ this.directive = FSImageSerialization.readCacheDirectiveInfo(st);
readRpcIdsFromXml(st);
}
@@ -3184,30 +3065,35 @@ public abstract class FSEditLogOp {
public AddCachePoolOp setPool(CachePoolInfo info) {
this.info = info;
+ assert(info.getPoolName() != null);
+ assert(info.getOwnerName() != null);
+ assert(info.getGroupName() != null);
+ assert(info.getMode() != null);
+ assert(info.getLimit() != null);
return this;
}
@Override
void readFields(DataInputStream in, int logVersion) throws IOException {
- info = CachePoolInfo.readFrom(in);
+ info = FSImageSerialization.readCachePoolInfo(in);
readRpcIds(in, logVersion);
}
@Override
public void writeFields(DataOutputStream out) throws IOException {
- info.writeTo(out);
+ FSImageSerialization.writeCachePoolInfo(out, info);
writeRpcIds(rpcClientId, rpcCallId, out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
- info.writeXmlTo(contentHandler);
+ FSImageSerialization.writeCachePoolInfo(contentHandler, info);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
- this.info = CachePoolInfo.readXmlFrom(st);
+ this.info = FSImageSerialization.readCachePoolInfo(st);
readRpcIdsFromXml(st);
}
@@ -3219,7 +3105,7 @@ public abstract class FSEditLogOp {
builder.append("ownerName=" + info.getOwnerName() + ",");
builder.append("groupName=" + info.getGroupName() + ",");
builder.append("mode=" + Short.toString(info.getMode().toShort()) + ",");
- builder.append("weight=" + Integer.toString(info.getWeight()));
+ builder.append("limit=" + Long.toString(info.getLimit()));
appendRpcIdsToString(builder, rpcClientId, rpcCallId);
builder.append("]");
return builder.toString();
@@ -3245,25 +3131,25 @@ public abstract class FSEditLogOp {
@Override
void readFields(DataInputStream in, int logVersion) throws IOException {
- info = CachePoolInfo.readFrom(in);
+ info = FSImageSerialization.readCachePoolInfo(in);
readRpcIds(in, logVersion);
}
@Override
public void writeFields(DataOutputStream out) throws IOException {
- info.writeTo(out);
+ FSImageSerialization.writeCachePoolInfo(out, info);
writeRpcIds(rpcClientId, rpcCallId, out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
- cachePoolInfoToXml(contentHandler, info);
+ FSImageSerialization.writeCachePoolInfo(contentHandler, info);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
- this.info = cachePoolInfoFromXml(st);
+ this.info = FSImageSerialization.readCachePoolInfo(st);
readRpcIdsFromXml(st);
}
@@ -3284,8 +3170,8 @@ public abstract class FSEditLogOp {
if (info.getMode() != null) {
fields.add("mode=" + info.getMode().toString());
}
- if (info.getWeight() != null) {
- fields.add("weight=" + info.getWeight());
+ if (info.getLimit() != null) {
+ fields.add("limit=" + info.getLimit());
}
builder.append(Joiner.on(",").join(fields));
appendRpcIdsToString(builder, rpcClientId, rpcCallId);
@@ -3439,9 +3325,7 @@ public abstract class FSEditLogOp {
* @param in The stream to read from.
* @param logVersion The version of the data coming from the stream.
*/
- @SuppressWarnings("deprecation")
- public Reader(DataInputStream in, StreamLimiter limiter,
- int logVersion) {
+ public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
this.logVersion = logVersion;
if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
this.checksum = new PureJavaCrc32();
@@ -3757,41 +3641,4 @@ public abstract class FSEditLogOp {
short mode = Short.valueOf(st.getValue("MODE"));
return new FsPermission(mode);
}
-
- public static void cachePoolInfoToXml(ContentHandler contentHandler,
- CachePoolInfo info) throws SAXException {
- XMLUtils.addSaxString(contentHandler, "POOLNAME", info.getPoolName());
- if (info.getOwnerName() != null) {
- XMLUtils.addSaxString(contentHandler, "OWNERNAME", info.getOwnerName());
- }
- if (info.getGroupName() != null) {
- XMLUtils.addSaxString(contentHandler, "GROUPNAME", info.getGroupName());
- }
- if (info.getMode() != null) {
- fsPermissionToXml(contentHandler, info.getMode());
- }
- if (info.getWeight() != null) {
- XMLUtils.addSaxString(contentHandler, "WEIGHT",
- Integer.toString(info.getWeight()));
- }
- }
-
- public static CachePoolInfo cachePoolInfoFromXml(Stanza st)
- throws InvalidXmlException {
- String poolName = st.getValue("POOLNAME");
- CachePoolInfo info = new CachePoolInfo(poolName);
- if (st.hasChildren("OWNERNAME")) {
- info.setOwnerName(st.getValue("OWNERNAME"));
- }
- if (st.hasChildren("GROUPNAME")) {
- info.setGroupName(st.getValue("GROUPNAME"));
- }
- if (st.hasChildren("MODE")) {
- info.setMode(FSEditLogOp.fsPermissionFromXml(st));
- }
- if (st.hasChildren("WEIGHT")) {
- info.setWeight(Integer.parseInt(st.getValue("WEIGHT")));
- }
- return info;
- }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Thu Dec 19 02:03:47 2013
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.util.Map;
-import java.util.HashMap;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -30,7 +27,6 @@ import org.apache.hadoop.classification.
@InterfaceStability.Unstable
public enum FSEditLogOpCodes {
// last op code in file
- OP_INVALID ((byte) -1),
OP_ADD ((byte) 0),
OP_RENAME_OLD ((byte) 1), // deprecated operation
OP_DELETE ((byte) 2),
@@ -69,9 +65,12 @@ public enum FSEditLogOpCodes {
OP_ADD_CACHE_POOL ((byte) 35),
OP_MODIFY_CACHE_POOL ((byte) 36),
OP_REMOVE_CACHE_POOL ((byte) 37),
- OP_MODIFY_CACHE_DIRECTIVE ((byte) 38);
+ OP_MODIFY_CACHE_DIRECTIVE ((byte) 38),
+
+ // Note that fromByte(..) depends on OP_INVALID being at the last position.
+ OP_INVALID ((byte) -1);
- private byte opCode;
+ private final byte opCode;
/**
* Constructor
@@ -91,14 +90,7 @@ public enum FSEditLogOpCodes {
return opCode;
}
- private static final Map<Byte, FSEditLogOpCodes> byteToEnum =
- new HashMap<Byte, FSEditLogOpCodes>();
-
- static {
- // initialize byte to enum map
- for(FSEditLogOpCodes opCode : values())
- byteToEnum.put(opCode.getOpCode(), opCode);
- }
+ private static final FSEditLogOpCodes[] VALUES = FSEditLogOpCodes.values();
/**
* Converts byte to FSEditLogOpCodes enum value
@@ -107,6 +99,12 @@ public enum FSEditLogOpCodes {
* @return enum with byte value of opCode
*/
public static FSEditLogOpCodes fromByte(byte opCode) {
- return byteToEnum.get(opCode);
+ if (opCode == -1) {
+ return OP_INVALID;
+ }
+ if (opCode >= 0 && opCode < OP_INVALID.ordinal()) {
+ return VALUES[opCode];
+ }
+ return null;
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Thu Dec 19 02:03:47 2013
@@ -52,9 +52,9 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
@@ -731,9 +731,10 @@ public class FSImageFormat {
if (nsQuota >= 0 || dsQuota >= 0) {
dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
}
- return snapshottable ? new INodeDirectorySnapshottable(dir)
- : withSnapshot ? new INodeDirectoryWithSnapshot(dir)
- : dir;
+ if (withSnapshot) {
+ dir.addSnapshotFeature(null);
+ }
+ return snapshottable ? new INodeDirectorySnapshottable(dir) : dir;
} else if (numBlocks == -2) {
//symlink
@@ -1113,10 +1114,10 @@ public class FSImageFormat {
final ReadOnlyList<INode> children = current.getChildrenList(null);
int dirNum = 0;
List<INodeDirectory> snapshotDirs = null;
- if (current instanceof INodeDirectoryWithSnapshot) {
+ DirectoryWithSnapshotFeature sf = current.getDirectoryWithSnapshotFeature();
+ if (sf != null) {
snapshotDirs = new ArrayList<INodeDirectory>();
- ((INodeDirectoryWithSnapshot) current).getSnapshotDirectory(
- snapshotDirs);
+ sf.getSnapshotDirectory(snapshotDirs);
dirNum += snapshotDirs.size();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Thu Dec 19 02:03:47 2013
@@ -30,20 +30,26 @@ import org.apache.hadoop.fs.permission.P
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
+import org.apache.hadoop.hdfs.util.XMLUtils;
+import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
+import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
import com.google.common.base.Preconditions;
@@ -239,7 +245,7 @@ public class FSImageSerialization {
out.writeBoolean(true);
} else {
out.writeBoolean(false);
- out.writeBoolean(node instanceof INodeDirectoryWithSnapshot);
+ out.writeBoolean(node.isWithSnapshot());
}
writePermissionStatus(node, out);
@@ -477,4 +483,202 @@ public class FSImageSerialization {
}
return ret;
}
+
+ public static void writeCacheDirectiveInfo(DataOutputStream out,
+ CacheDirectiveInfo directive) throws IOException {
+ writeLong(directive.getId(), out);
+ int flags =
+ ((directive.getPath() != null) ? 0x1 : 0) |
+ ((directive.getReplication() != null) ? 0x2 : 0) |
+ ((directive.getPool() != null) ? 0x4 : 0) |
+ ((directive.getExpiration() != null) ? 0x8 : 0);
+ out.writeInt(flags);
+ if (directive.getPath() != null) {
+ writeString(directive.getPath().toUri().getPath(), out);
+ }
+ if (directive.getReplication() != null) {
+ writeShort(directive.getReplication(), out);
+ }
+ if (directive.getPool() != null) {
+ writeString(directive.getPool(), out);
+ }
+ if (directive.getExpiration() != null) {
+ writeLong(directive.getExpiration().getMillis(), out);
+ }
+ }
+
+ public static CacheDirectiveInfo readCacheDirectiveInfo(DataInput in)
+ throws IOException {
+ CacheDirectiveInfo.Builder builder =
+ new CacheDirectiveInfo.Builder();
+ builder.setId(readLong(in));
+ int flags = in.readInt();
+ if ((flags & 0x1) != 0) {
+ builder.setPath(new Path(readString(in)));
+ }
+ if ((flags & 0x2) != 0) {
+ builder.setReplication(readShort(in));
+ }
+ if ((flags & 0x4) != 0) {
+ builder.setPool(readString(in));
+ }
+ if ((flags & 0x8) != 0) {
+ builder.setExpiration(
+ CacheDirectiveInfo.Expiration.newAbsolute(readLong(in)));
+ }
+ if ((flags & ~0xF) != 0) {
+ throw new IOException("unknown flags set in " +
+ "ModifyCacheDirectiveInfoOp: " + flags);
+ }
+ return builder.build();
+ }
+
+ public static CacheDirectiveInfo readCacheDirectiveInfo(Stanza st)
+ throws InvalidXmlException {
+ CacheDirectiveInfo.Builder builder =
+ new CacheDirectiveInfo.Builder();
+ builder.setId(Long.parseLong(st.getValue("ID")));
+ String path = st.getValueOrNull("PATH");
+ if (path != null) {
+ builder.setPath(new Path(path));
+ }
+ String replicationString = st.getValueOrNull("REPLICATION");
+ if (replicationString != null) {
+ builder.setReplication(Short.parseShort(replicationString));
+ }
+ String pool = st.getValueOrNull("POOL");
+ if (pool != null) {
+ builder.setPool(pool);
+ }
+ String expiryTime = st.getValueOrNull("EXPIRATION");
+ if (expiryTime != null) {
+ builder.setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(
+ Long.parseLong(expiryTime)));
+ }
+ return builder.build();
+ }
+
+ public static void writeCacheDirectiveInfo(ContentHandler contentHandler,
+ CacheDirectiveInfo directive) throws SAXException {
+ XMLUtils.addSaxString(contentHandler, "ID",
+ Long.toString(directive.getId()));
+ if (directive.getPath() != null) {
+ XMLUtils.addSaxString(contentHandler, "PATH",
+ directive.getPath().toUri().getPath());
+ }
+ if (directive.getReplication() != null) {
+ XMLUtils.addSaxString(contentHandler, "REPLICATION",
+ Short.toString(directive.getReplication()));
+ }
+ if (directive.getPool() != null) {
+ XMLUtils.addSaxString(contentHandler, "POOL", directive.getPool());
+ }
+ if (directive.getExpiration() != null) {
+ XMLUtils.addSaxString(contentHandler, "EXPIRATION",
+ "" + directive.getExpiration().getMillis());
+ }
+ }
+
+ public static void writeCachePoolInfo(DataOutputStream out, CachePoolInfo info)
+ throws IOException {
+ writeString(info.getPoolName(), out);
+
+ final String ownerName = info.getOwnerName();
+ final String groupName = info.getGroupName();
+ final Long limit = info.getLimit();
+ final FsPermission mode = info.getMode();
+
+ boolean hasOwner, hasGroup, hasMode, hasLimit;
+ hasOwner = ownerName != null;
+ hasGroup = groupName != null;
+ hasMode = mode != null;
+ hasLimit = limit != null;
+
+ int flags =
+ (hasOwner ? 0x1 : 0) |
+ (hasGroup ? 0x2 : 0) |
+ (hasMode ? 0x4 : 0) |
+ (hasLimit ? 0x8 : 0);
+ writeInt(flags, out);
+
+ if (hasOwner) {
+ writeString(ownerName, out);
+ }
+ if (hasGroup) {
+ writeString(groupName, out);
+ }
+ if (hasMode) {
+ mode.write(out);
+ }
+ if (hasLimit) {
+ writeLong(limit, out);
+ }
+ }
+
+ public static CachePoolInfo readCachePoolInfo(DataInput in)
+ throws IOException {
+ String poolName = readString(in);
+ CachePoolInfo info = new CachePoolInfo(poolName);
+ int flags = readInt(in);
+ if ((flags & 0x1) != 0) {
+ info.setOwnerName(readString(in));
+ }
+ if ((flags & 0x2) != 0) {
+ info.setGroupName(readString(in));
+ }
+ if ((flags & 0x4) != 0) {
+ info.setMode(FsPermission.read(in));
+ }
+ if ((flags & 0x8) != 0) {
+ info.setLimit(readLong(in));
+ }
+ if ((flags & ~0xF) != 0) {
+ throw new IOException("Unknown flag in CachePoolInfo: " + flags);
+ }
+ return info;
+ }
+
+ public static void writeCachePoolInfo(ContentHandler contentHandler,
+ CachePoolInfo info) throws SAXException {
+ XMLUtils.addSaxString(contentHandler, "POOLNAME", info.getPoolName());
+
+ final String ownerName = info.getOwnerName();
+ final String groupName = info.getGroupName();
+ final Long limit = info.getLimit();
+ final FsPermission mode = info.getMode();
+
+ if (ownerName != null) {
+ XMLUtils.addSaxString(contentHandler, "OWNERNAME", ownerName);
+ }
+ if (groupName != null) {
+ XMLUtils.addSaxString(contentHandler, "GROUPNAME", groupName);
+ }
+ if (mode != null) {
+ FSEditLogOp.fsPermissionToXml(contentHandler, mode);
+ }
+ if (limit != null) {
+ XMLUtils.addSaxString(contentHandler, "LIMIT",
+ Long.toString(limit));
+ }
+ }
+
+ public static CachePoolInfo readCachePoolInfo(Stanza st)
+ throws InvalidXmlException {
+ String poolName = st.getValue("POOLNAME");
+ CachePoolInfo info = new CachePoolInfo(poolName);
+ if (st.hasChildren("OWNERNAME")) {
+ info.setOwnerName(st.getValue("OWNERNAME"));
+ }
+ if (st.hasChildren("GROUPNAME")) {
+ info.setGroupName(st.getValue("GROUPNAME"));
+ }
+ if (st.hasChildren("MODE")) {
+ info.setMode(FSEditLogOp.fsPermissionFromXml(st));
+ }
+ if (st.hasChildren("LIMIT")) {
+ info.setLimit(Long.parseLong(st.getValue("LIMIT")));
+ }
+ return info;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Dec 19 02:03:47 2013
@@ -126,6 +126,7 @@ import org.apache.hadoop.HadoopIllegalAr
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
@@ -735,12 +736,7 @@ public class FSNamesystem implements Nam
this.dtSecretManager = createDelegationTokenSecretManager(conf);
this.dir = new FSDirectory(fsImage, this, conf);
this.snapshotManager = new SnapshotManager(dir);
- writeLock();
- try {
- this.cacheManager = new CacheManager(this, conf, blockManager);
- } finally {
- writeUnlock();
- }
+ this.cacheManager = new CacheManager(this, conf, blockManager);
this.safeMode = new SafeModeInfo(conf);
this.auditLoggers = initAuditLoggers(conf);
this.isDefaultAuditLogger = auditLoggers.size() == 1 &&
@@ -2295,7 +2291,7 @@ public class FSNamesystem implements Nam
String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
boolean writeToEditLog, Snapshot latestSnapshot, boolean logRetryCache)
throws IOException {
- file = file.recordModification(latestSnapshot, dir.getINodeMap());
+ file = file.recordModification(latestSnapshot);
final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine,
clientNode);
@@ -2877,8 +2873,10 @@ public class FSNamesystem implements Nam
writeUnlock();
}
getEditLog().logSync();
- NameNode.stateChangeLog.info("DIR* completeFile: " + src + " is closed by "
- + holder);
+ if (success) {
+ NameNode.stateChangeLog.info("DIR* completeFile: " + src
+ + " is closed by " + holder);
+ }
return success;
}
@@ -3783,8 +3781,7 @@ public class FSNamesystem implements Nam
Preconditions.checkArgument(uc != null);
leaseManager.removeLease(uc.getClientName(), src);
- pendingFile = pendingFile.recordModification(latestSnapshot,
- dir.getINodeMap());
+ pendingFile = pendingFile.recordModification(latestSnapshot);
// The file is no longer pending.
// Create permanent INode, update blocks. No need to replace the inode here
@@ -7056,8 +7053,8 @@ public class FSNamesystem implements Nam
}
}
- long addCacheDirective(
- CacheDirectiveInfo directive) throws IOException {
+ long addCacheDirective(CacheDirectiveInfo directive, EnumSet<CacheFlag> flags)
+ throws IOException {
checkOperation(OperationCategory.WRITE);
final FSPermissionChecker pc = isPermissionEnabled ?
getPermissionChecker() : null;
@@ -7080,7 +7077,7 @@ public class FSNamesystem implements Nam
"for this operation.");
}
CacheDirectiveInfo effectiveDirective =
- cacheManager.addDirective(directive, pc);
+ cacheManager.addDirective(directive, pc, flags);
getEditLog().logAddCacheDirectiveInfo(effectiveDirective,
cacheEntry != null);
result = effectiveDirective.getId();
@@ -7098,8 +7095,8 @@ public class FSNamesystem implements Nam
return result;
}
- void modifyCacheDirective(
- CacheDirectiveInfo directive) throws IOException {
+ void modifyCacheDirective(CacheDirectiveInfo directive,
+ EnumSet<CacheFlag> flags) throws IOException {
checkOperation(OperationCategory.WRITE);
final FSPermissionChecker pc = isPermissionEnabled ?
getPermissionChecker() : null;
@@ -7115,7 +7112,7 @@ public class FSNamesystem implements Nam
throw new SafeModeException(
"Cannot add cache directive", safeMode);
}
- cacheManager.modifyDirective(directive, pc);
+ cacheManager.modifyDirective(directive, pc, flags);
getEditLog().logModifyCacheDirectiveInfo(directive,
cacheEntry != null);
success = true;
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java Thu Dec 19 02:03:47 2013
@@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.server.nam
* Feature for under-construction file.
*/
@InterfaceAudience.Private
-public class FileUnderConstructionFeature extends INodeFile.Feature {
+public class FileUnderConstructionFeature implements INode.Feature {
private String clientName; // lease holder
private final String clientMachine;
// if client is a cluster node too.
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java Thu Dec 19 02:03:47 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.util.Time.now;
+
import java.security.PrivilegedExceptionAction;
import java.util.*;
import java.io.*;
@@ -41,6 +43,7 @@ import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
@@ -88,6 +91,7 @@ public class GetImageServlet extends Htt
final GetImageParams parsedParams = new GetImageParams(request, response);
final Configuration conf = (Configuration) context
.getAttribute(JspHelper.CURRENT_CONF);
+ final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
if (UserGroupInformation.isSecurityEnabled() &&
!isValidRequestor(context, request.getUserPrincipal().getName(), conf)) {
@@ -128,14 +132,26 @@ public class GetImageServlet extends Htt
throw new IOException(errorMessage);
}
CheckpointFaultInjector.getInstance().beforeGetImageSetsHeaders();
+ long start = now();
serveFile(imageFile);
+
+ if (metrics != null) { // Metrics non-null only when used inside name node
+ long elapsed = now() - start;
+ metrics.addGetImage(elapsed);
+ }
} else if (parsedParams.isGetEdit()) {
long startTxId = parsedParams.getStartTxId();
long endTxId = parsedParams.getEndTxId();
File editFile = nnImage.getStorage()
.findFinalizedEditsFile(startTxId, endTxId);
+ long start = now();
serveFile(editFile);
+
+ if (metrics != null) { // Metrics non-null only when used inside name node
+ long elapsed = now() - start;
+ metrics.addGetEdit(elapsed);
+ }
} else if (parsedParams.isPutImage()) {
final long txid = parsedParams.getTxId();
@@ -159,12 +175,18 @@ public class GetImageServlet extends Htt
UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
}
+ long start = now();
// issue a HTTP get request to download the new fsimage
MD5Hash downloadImageDigest =
TransferFsImage.downloadImageToStorage(
parsedParams.getInfoServer(conf), txid,
nnImage.getStorage(), true);
nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest);
+
+ if (metrics != null) { // Metrics non-null only when used inside name node
+ long elapsed = now() - start;
+ metrics.addPutImage(elapsed);
+ }
// Now that we have a new checkpoint, we might be able to
// remove some old ones.
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1552205&r1=1552204&r2=1552205&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Thu Dec 19 02:03:47 2013
@@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.Diff;
@@ -96,9 +95,9 @@ public abstract class INode implements I
abstract void setUser(String user);
/** Set user */
- final INode setUser(String user, Snapshot latest, INodeMap inodeMap)
+ final INode setUser(String user, Snapshot latest)
throws QuotaExceededException {
- final INode nodeToUpdate = recordModification(latest, inodeMap);
+ final INode nodeToUpdate = recordModification(latest);
nodeToUpdate.setUser(user);
return nodeToUpdate;
}
@@ -120,9 +119,9 @@ public abstract class INode implements I
abstract void setGroup(String group);
/** Set group */
- final INode setGroup(String group, Snapshot latest, INodeMap inodeMap)
+ final INode setGroup(String group, Snapshot latest)
throws QuotaExceededException {
- final INode nodeToUpdate = recordModification(latest, inodeMap);
+ final INode nodeToUpdate = recordModification(latest);
nodeToUpdate.setGroup(group);
return nodeToUpdate;
}
@@ -145,9 +144,9 @@ public abstract class INode implements I
abstract void setPermission(FsPermission permission);
/** Set the {@link FsPermission} of this {@link INode} */
- INode setPermission(FsPermission permission, Snapshot latest,
- INodeMap inodeMap) throws QuotaExceededException {
- final INode nodeToUpdate = recordModification(latest, inodeMap);
+ INode setPermission(FsPermission permission, Snapshot latest)
+ throws QuotaExceededException {
+ final INode nodeToUpdate = recordModification(latest);
nodeToUpdate.setPermission(permission);
return nodeToUpdate;
}
@@ -231,14 +230,12 @@ public abstract class INode implements I
*
* @param latest the latest snapshot that has been taken.
* Note that it is null if no snapshots have been taken.
- * @param inodeMap while recording modification, the inode or its parent may
- * get replaced, and the inodeMap needs to be updated.
* @return The current inode, which usually is the same object of this inode.
* However, in some cases, this inode may be replaced with a new inode
* for maintaining snapshots. The current inode is then the new inode.
*/
- abstract INode recordModification(final Snapshot latest,
- final INodeMap inodeMap) throws QuotaExceededException;
+ abstract INode recordModification(final Snapshot latest)
+ throws QuotaExceededException;
/** Check whether it's a reference. */
public boolean isReference() {
@@ -318,7 +315,7 @@ public abstract class INode implements I
* Call recordModification(..) to capture the current states.
* Mark the INode as deleted.
*
- * 1.4 The current inode is a {@link INodeDirectoryWithSnapshot}.
+ * 1.4 The current inode is an {@link INodeDirectory} with snapshot feature.
* Call recordModification(..) to capture the current states.
* Destroy files/directories created after the latest snapshot
* (i.e., the inodes stored in the created list of the latest snapshot).
@@ -329,7 +326,7 @@ public abstract class INode implements I
* 2.2 To clean {@link INodeDirectory}: recursively clean its children.
* 2.3 To clean INodeFile with snapshot: delete the corresponding snapshot in
* its diff list.
- * 2.4 To clean {@link INodeDirectoryWithSnapshot}: delete the corresponding
+ * 2.4 To clean {@link INodeDirectory} with snapshot: delete the corresponding
* snapshot in its diff list. Recursively clean its children.
* </pre>
*
@@ -575,16 +572,16 @@ public abstract class INode implements I
}
/** Update modification time if it is larger than the current value. */
- public abstract INode updateModificationTime(long mtime, Snapshot latest,
- INodeMap inodeMap) throws QuotaExceededException;
+ public abstract INode updateModificationTime(long mtime, Snapshot latest)
+ throws QuotaExceededException;
/** Set the last modification time of inode. */
public abstract void setModificationTime(long modificationTime);
/** Set the last modification time of inode. */
public final INode setModificationTime(long modificationTime,
- Snapshot latest, INodeMap inodeMap) throws QuotaExceededException {
- final INode nodeToUpdate = recordModification(latest, inodeMap);
+ Snapshot latest) throws QuotaExceededException {
+ final INode nodeToUpdate = recordModification(latest);
nodeToUpdate.setModificationTime(modificationTime);
return nodeToUpdate;
}
@@ -611,9 +608,9 @@ public abstract class INode implements I
/**
* Set last access time of inode.
*/
- public final INode setAccessTime(long accessTime, Snapshot latest,
- INodeMap inodeMap) throws QuotaExceededException {
- final INode nodeToUpdate = recordModification(latest, inodeMap);
+ public final INode setAccessTime(long accessTime, Snapshot latest)
+ throws QuotaExceededException {
+ final INode nodeToUpdate = recordModification(latest);
nodeToUpdate.setAccessTime(accessTime);
return nodeToUpdate;
}
@@ -753,50 +750,10 @@ public abstract class INode implements I
}
}
- /** INode feature such as {@link FileUnderConstructionFeature}
- * and {@link DirectoryWithQuotaFeature}.
+ /**
+ * INode feature such as {@link FileUnderConstructionFeature}
+ * and {@link DirectoryWithQuotaFeature}.
*/
- interface Feature<F extends Feature<F>> {
- /** @return the next feature. */
- public F getNextFeature();
-
- /** Set the next feature. */
- public void setNextFeature(F next);
-
- /** Utility methods such as addFeature and removeFeature. */
- static class Util {
- /**
- * Add a feature to the linked list.
- * @return the new head.
- */
- static <F extends Feature<F>> F addFeature(F feature, F head) {
- feature.setNextFeature(head);
- return feature;
- }
-
- /**
- * Remove a feature from the linked list.
- * @return the new head.
- */
- static <F extends Feature<F>> F removeFeature(F feature, F head) {
- if (feature == head) {
- final F newHead = head.getNextFeature();
- head.setNextFeature(null);
- return newHead;
- } else if (head != null) {
- F prev = head;
- F curr = head.getNextFeature();
- for (; curr != null && curr != feature;
- prev = curr, curr = curr.getNextFeature())
- ;
- if (curr != null) {
- prev.setNextFeature(curr.getNextFeature());
- curr.setNextFeature(null);
- return head;
- }
- }
- throw new IllegalStateException("Feature " + feature + " not found.");
- }
- }
+ public interface Feature {
}
}