You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/09/05 21:07:04 UTC
[14/20] git commit: HDFS-6886. Use single editlog record for creating
file + overwrite. Contributed by Yi Liu.
HDFS-6886. Use single editlog record for creating file + overwrite. Contributed by Yi Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61045203
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61045203
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61045203
Branch: refs/heads/HDFS-6584
Commit: 6104520369045dfaa4b543cbad21236ed322249b
Parents: f7df24b
Author: Jing Zhao <ji...@hortonworks.com>
Authored: Thu Sep 4 18:54:38 2014 -0700
Committer: Jing Zhao <ji...@hortonworks.com>
Committed: Thu Sep 4 18:54:38 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/inotify/Event.java | 12 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 8 +-
.../hdfs/server/namenode/FSEditLogOp.java | 18 ++
.../server/namenode/FSImageSerialization.java | 17 ++
.../hdfs/server/namenode/FSNamesystem.java | 45 +++--
.../namenode/InotifyFSEditLogOpTranslator.java | 1 +
.../server/namenode/NameNodeLayoutVersion.java | 4 +-
.../hadoop-hdfs/src/main/proto/inotify.proto | 1 +
.../hdfs/TestDFSInotifyEventInputStream.java | 8 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 119 ++++++++++++
.../hdfs/server/namenode/CreateEditsLog.java | 2 +-
.../hdfs/server/namenode/TestEditLog.java | 2 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 4992 -> 5252 bytes
.../src/test/resources/editsStored.xml | 182 ++++++++++++-------
17 files changed, 339 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3e00eba..8964d2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -438,6 +438,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6959. Make the HDFS home directory location customizable. (yzhang via
cmccabe)
+ HDFS-6886. Use single editlog record for creating file + overwrite. (Yi Liu
+ via jing9)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index c7129ca..e8a34e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -100,6 +100,7 @@ public abstract class Event {
private String groupName;
private FsPermission perms;
private String symlinkTarget;
+ private boolean overwrite;
public static class Builder {
private INodeType iNodeType;
@@ -110,6 +111,7 @@ public abstract class Event {
private String groupName;
private FsPermission perms;
private String symlinkTarget;
+ private boolean overwrite;
public Builder iNodeType(INodeType type) {
this.iNodeType = type;
@@ -150,6 +152,11 @@ public abstract class Event {
this.symlinkTarget = symlinkTarget;
return this;
}
+
+ public Builder overwrite(boolean overwrite) {
+ this.overwrite = overwrite;
+ return this;
+ }
public CreateEvent build() {
return new CreateEvent(this);
@@ -166,6 +173,7 @@ public abstract class Event {
this.groupName = b.groupName;
this.perms = b.perms;
this.symlinkTarget = b.symlinkTarget;
+ this.overwrite = b.overwrite;
}
public INodeType getiNodeType() {
@@ -208,6 +216,10 @@ public abstract class Event {
public String getSymlinkTarget() {
return symlinkTarget;
}
+
+ public boolean getOverwrite() {
+ return overwrite;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 38ba7db..193b826 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -2430,7 +2430,8 @@ public class PBHelper {
.perms(convert(create.getPerms()))
.replication(create.getReplication())
.symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
- create.getSymlinkTarget()).build());
+ create.getSymlinkTarget())
+ .overwrite(create.getOverwrite()).build());
break;
case EVENT_METADATA:
InotifyProtos.MetadataUpdateEventProto meta =
@@ -2508,7 +2509,8 @@ public class PBHelper {
.setPerms(convert(ce2.getPerms()))
.setReplication(ce2.getReplication())
.setSymlinkTarget(ce2.getSymlinkTarget() == null ?
- "" : ce2.getSymlinkTarget()).build().toByteString()
+ "" : ce2.getSymlinkTarget())
+ .setOverwrite(ce2.getOverwrite()).build().toByteString()
).build());
break;
case METADATA:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 9d3538d..a157f4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -702,7 +702,8 @@ public class FSEditLog implements LogsPurgeable {
* Add open lease record to edit log.
* Records the block locations of the last block.
*/
- public void logOpenFile(String path, INodeFile newNode, boolean toLogRpcIds) {
+ public void logOpenFile(String path, INodeFile newNode, boolean overwrite,
+ boolean toLogRpcIds) {
Preconditions.checkArgument(newNode.isUnderConstruction());
PermissionStatus permissions = newNode.getPermissionStatus();
AddOp op = AddOp.getInstance(cache.get())
@@ -716,7 +717,8 @@ public class FSEditLog implements LogsPurgeable {
.setPermissionStatus(permissions)
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
.setClientMachine(
- newNode.getFileUnderConstructionFeature().getClientMachine());
+ newNode.getFileUnderConstructionFeature().getClientMachine())
+ .setOverwrite(overwrite);
AclFeature f = newNode.getAclFeature();
if (f != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index d522e51..6afd81c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -340,8 +340,12 @@ public class FSEditLogLoader {
// See if the file already exists (persistBlocks call)
final INodesInPath iip = fsDir.getLastINodeInPath(path);
- final INodeFile oldFile = INodeFile.valueOf(
- iip.getINode(0), path, true);
+ INodeFile oldFile = INodeFile.valueOf(iip.getINode(0), path, true);
+ if (oldFile != null && addCloseOp.overwrite) {
+ // This is OP_ADD with overwrite
+ fsDir.unprotectedDelete(path, addCloseOp.mtime);
+ oldFile = null;
+ }
INodeFile newFile = oldFile;
if (oldFile == null) { // this is OP_ADD on a new file (case 1)
// versions > 0 support per file replication
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 5543e0c..c5dd3df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -407,6 +407,7 @@ public abstract class FSEditLogOp {
List<XAttr> xAttrs;
String clientName;
String clientMachine;
+ boolean overwrite;
private AddCloseOp(FSEditLogOpCodes opCode) {
super(opCode);
@@ -486,6 +487,11 @@ public abstract class FSEditLogOp {
this.clientMachine = clientMachine;
return (T)this;
}
+
+ <T extends AddCloseOp> T setOverwrite(boolean overwrite) {
+ this.overwrite = overwrite;
+ return (T)this;
+ }
@Override
public void writeFields(DataOutputStream out) throws IOException {
@@ -505,6 +511,7 @@ public abstract class FSEditLogOp {
b.build().writeDelimitedTo(out);
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
+ FSImageSerialization.writeBoolean(overwrite, out);
// write clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
}
@@ -570,6 +577,12 @@ public abstract class FSEditLogOp {
this.xAttrs = readXAttrsFromEditLog(in, logVersion);
this.clientName = FSImageSerialization.readString(in);
this.clientMachine = FSImageSerialization.readString(in);
+ if (NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.CREATE_OVERWRITE, logVersion)) {
+ this.overwrite = FSImageSerialization.readBoolean(in);
+ } else {
+ this.overwrite = false;
+ }
// read clientId and callId
readRpcIds(in, logVersion);
} else {
@@ -625,6 +638,8 @@ public abstract class FSEditLogOp {
builder.append(clientName);
builder.append(", clientMachine=");
builder.append(clientMachine);
+ builder.append(", overwrite=");
+ builder.append(overwrite);
if (this.opCode == OP_ADD) {
appendRpcIdsToString(builder, rpcClientId, rpcCallId);
}
@@ -653,6 +668,8 @@ public abstract class FSEditLogOp {
Long.toString(blockSize));
XMLUtils.addSaxString(contentHandler, "CLIENT_NAME", clientName);
XMLUtils.addSaxString(contentHandler, "CLIENT_MACHINE", clientMachine);
+ XMLUtils.addSaxString(contentHandler, "OVERWRITE",
+ Boolean.toString(overwrite));
for (Block b : blocks) {
FSEditLogOp.blockToXml(contentHandler, b);
}
@@ -676,6 +693,7 @@ public abstract class FSEditLogOp {
this.blockSize = Long.parseLong(st.getValue("BLOCKSIZE"));
this.clientName = st.getValue("CLIENT_NAME");
this.clientMachine = st.getValue("CLIENT_MACHINE");
+ this.overwrite = Boolean.parseBoolean(st.getValueOrNull("OVERWRITE"));
if (st.hasChildren("BLOCK")) {
List<Stanza> blocks = st.getChildren("BLOCK");
this.blocks = new Block[blocks.size()];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index eb8354d..3956d9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.Ref
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
+import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ShortWritable;
@@ -88,6 +89,7 @@ public class FSImageSerialization {
final IntWritable U_INT = new IntWritable();
final LongWritable U_LONG = new LongWritable();
final FsPermission FILE_PERM = new FsPermission((short) 0);
+ final BooleanWritable U_BOOLEAN = new BooleanWritable();
}
private static void writePermissionStatus(INodeAttributes inode,
@@ -366,6 +368,21 @@ public class FSImageSerialization {
uLong.write(out);
}
+ /** read the boolean value */
+ static boolean readBoolean(DataInput in) throws IOException {
+ BooleanWritable uBoolean = TL_DATA.get().U_BOOLEAN;
+ uBoolean.readFields(in);
+ return uBoolean.get();
+ }
+
+ /** write the boolean value */
+ static void writeBoolean(boolean value, DataOutputStream out)
+ throws IOException {
+ BooleanWritable uBoolean = TL_DATA.get().U_BOOLEAN;
+ uBoolean.set(value);
+ uBoolean.write(out);
+ }
+
/** read the int value */
static int readInt(DataInput in) throws IOException {
IntWritable uInt = TL_DATA.get().U_INT;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 5d60dd7..c1744f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2448,6 +2448,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
* A special RetryStartFileException is used to indicate that we should
* retry creation of a FileEncryptionInfo.
*/
+ BlocksMapUpdateInfo toRemoveBlocks = null;
try {
boolean shouldContinue = true;
int iters = 0;
@@ -2496,9 +2497,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot create file" + src);
src = resolvePath(src, pathComponents);
- startFileInternal(pc, src, permissions, holder, clientMachine, create,
- overwrite, createParent, replication, blockSize, suite, edek,
- logRetryCache);
+ toRemoveBlocks = startFileInternal(pc, src, permissions, holder,
+ clientMachine, create, overwrite, createParent, replication,
+ blockSize, suite, edek, logRetryCache);
stat = dir.getFileInfo(src, false,
FSDirectory.isReservedRawName(srcArg));
} catch (StandbyException se) {
@@ -2519,6 +2520,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
// They need to be sync'ed even when an exception was thrown.
if (!skipSync) {
getEditLog().logSync();
+ if (toRemoveBlocks != null) {
+ removeBlocks(toRemoveBlocks);
+ toRemoveBlocks.clear();
+ }
}
}
@@ -2535,11 +2540,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
* For description of parameters and exceptions thrown see
* {@link ClientProtocol#create}
*/
- private void startFileInternal(FSPermissionChecker pc, String src,
- PermissionStatus permissions, String holder, String clientMachine,
- boolean create, boolean overwrite, boolean createParent,
- short replication, long blockSize, CipherSuite suite,
- EncryptedKeyVersion edek, boolean logRetryEntry)
+ private BlocksMapUpdateInfo startFileInternal(FSPermissionChecker pc,
+ String src, PermissionStatus permissions, String holder,
+ String clientMachine, boolean create, boolean overwrite,
+ boolean createParent, short replication, long blockSize,
+ CipherSuite suite, EncryptedKeyVersion edek, boolean logRetryEntry)
throws FileAlreadyExistsException, AccessControlException,
UnresolvedLinkException, FileNotFoundException,
ParentNotDirectoryException, RetryStartFileException, IOException {
@@ -2575,9 +2580,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
if (isPermissionEnabled) {
if (overwrite && myFile != null) {
checkPathAccess(pc, src, FsAction.WRITE);
- } else {
- checkAncestorAccess(pc, src, FsAction.WRITE);
}
+ /*
+ * To overwrite existing file, need to check 'w' permission
+ * of parent (equals to ancestor in this case)
+ */
+ checkAncestorAccess(pc, src, FsAction.WRITE);
}
if (!createParent) {
@@ -2585,6 +2593,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
try {
+ BlocksMapUpdateInfo toRemoveBlocks = null;
if (myFile == null) {
if (!create) {
throw new FileNotFoundException("Can't overwrite non-existent " +
@@ -2592,11 +2601,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
} else {
if (overwrite) {
- try {
- deleteInt(src, true, false); // File exists - delete if overwrite
- } catch (AccessControlException e) {
- logAuditEvent(false, "delete", src);
- throw e;
+ toRemoveBlocks = new BlocksMapUpdateInfo();
+ List<INode> toRemoveINodes = new ChunkedArrayList<INode>();
+ long ret = dir.delete(src, toRemoveBlocks, toRemoveINodes, now());
+ if (ret >= 0) {
+ incrDeletedFileCount(ret);
+ removePathAndBlocks(src, null, toRemoveINodes, true);
}
} else {
// If lease soft limit time is expired, recover the lease
@@ -2630,11 +2640,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
// record file record in log, record new generation stamp
- getEditLog().logOpenFile(src, newNode, logRetryEntry);
+ getEditLog().logOpenFile(src, newNode, overwrite, logRetryEntry);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
+ return toRemoveBlocks;
} catch (IOException ie) {
NameNode.stateChangeLog.warn("DIR* NameSystem.startFile: " + src + " " +
ie.getMessage());
@@ -2737,7 +2748,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
if (writeToEditLog) {
- getEditLog().logOpenFile(src, cons, logRetryCache);
+ getEditLog().logOpenFile(src, cons, false, logRetryCache);
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
index 676f887..00a5f25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
@@ -50,6 +50,7 @@ public class InotifyFSEditLogOpTranslator {
.ownerName(addOp.permissions.getUserName())
.groupName(addOp.permissions.getGroupName())
.perms(addOp.permissions.getPermission())
+ .overwrite(addOp.overwrite)
.iNodeType(Event.CreateEvent.INodeType.FILE).build() };
} else {
return new Event[] { new Event.AppendEvent(addOp.path) };
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 6ae2806..1df6df4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -65,7 +65,9 @@ public class NameNodeLayoutVersion {
public static enum Feature implements LayoutFeature {
ROLLING_UPGRADE(-55, -53, "Support rolling upgrade", false),
EDITLOG_LENGTH(-56, "Add length field to every edit log op"),
- XATTRS(-57, "Extended attributes");
+ XATTRS(-57, "Extended attributes"),
+ CREATE_OVERWRITE(-58, "Use single editlog record for " +
+ "creating file with overwrite");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
index b58bfcc..a1d4d92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
@@ -72,6 +72,7 @@ message CreateEventProto {
required FsPermissionProto perms = 6;
optional int32 replication = 7;
optional string symlinkTarget = 8;
+ optional bool overwrite = 9;
}
message CloseEventProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index c268281..45b588b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -147,12 +147,7 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(re2.getSrcPath().equals("/file4"));
Assert.assertTrue(re.getTimestamp() > 0);
- // DeleteOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.UNLINK);
- Assert.assertTrue(((Event.UnlinkEvent) next).getPath().equals("/file2"));
-
- // AddOp
+ // AddOp with overwrite
next = waitForNextEvent(eis);
Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
Event.CreateEvent ce = (Event.CreateEvent) next;
@@ -161,6 +156,7 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce.getCtime() > 0);
Assert.assertTrue(ce.getReplication() > 0);
Assert.assertTrue(ce.getSymlinkTarget() == null);
+ Assert.assertTrue(ce.getOverwrite());
// CloseOp
next = waitForNextEvent(eis);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index c8b7df2..3a399f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -38,6 +38,7 @@ import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
@@ -70,6 +71,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
@@ -78,6 +80,8 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
@@ -86,6 +90,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -1210,4 +1215,118 @@ public class TestFileCreation {
}
}
+ /**
+ * 1. Check the blocks of old file are cleaned after creating with overwrite
+ * 2. Restart NN, check the file
+ * 3. Save new checkpoint and restart NN, check the file
+ */
+ @Test(timeout = 120000)
+ public void testFileCreationWithOverwrite() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt("dfs.blocksize", blockSize);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).
+ numDataNodes(3).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ try {
+ dfs.mkdirs(new Path("/foo/dir"));
+ String file = "/foo/dir/file";
+ Path filePath = new Path(file);
+
+ // Case 1: Create file with overwrite, check the blocks of old file
+ // are cleaned after creating with overwrite
+ NameNode nn = cluster.getNameNode();
+ FSNamesystem fsn = NameNodeAdapter.getNamesystem(nn);
+ BlockManager bm = fsn.getBlockManager();
+
+ FSDataOutputStream out = dfs.create(filePath);
+ byte[] oldData = AppendTestUtil.randomBytes(seed, fileSize);
+ try {
+ out.write(oldData);
+ } finally {
+ out.close();
+ }
+
+ LocatedBlocks oldBlocks = NameNodeAdapter.getBlockLocations(
+ nn, file, 0, fileSize);
+ assertBlocks(bm, oldBlocks, true);
+
+ out = dfs.create(filePath, true);
+ byte[] newData = AppendTestUtil.randomBytes(seed, fileSize);
+ try {
+ out.write(newData);
+ } finally {
+ out.close();
+ }
+ dfs.deleteOnExit(filePath);
+
+ LocatedBlocks newBlocks = NameNodeAdapter.getBlockLocations(
+ nn, file, 0, fileSize);
+ assertBlocks(bm, newBlocks, true);
+ assertBlocks(bm, oldBlocks, false);
+
+ FSDataInputStream in = dfs.open(filePath);
+ byte[] result = null;
+ try {
+ result = readAll(in);
+ } finally {
+ in.close();
+ }
+ Assert.assertArrayEquals(newData, result);
+
+ // Case 2: Restart NN, check the file
+ cluster.restartNameNode();
+ nn = cluster.getNameNode();
+ in = dfs.open(filePath);
+ try {
+ result = readAll(in);
+ } finally {
+ in.close();
+ }
+ Assert.assertArrayEquals(newData, result);
+
+ // Case 3: Save new checkpoint and restart NN, check the file
+ NameNodeAdapter.enterSafeMode(nn, false);
+ NameNodeAdapter.saveNamespace(nn);
+ cluster.restartNameNode();
+ nn = cluster.getNameNode();
+
+ in = dfs.open(filePath);
+ try {
+ result = readAll(in);
+ } finally {
+ in.close();
+ }
+ Assert.assertArrayEquals(newData, result);
+ } finally {
+ if (dfs != null) {
+ dfs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private void assertBlocks(BlockManager bm, LocatedBlocks lbs,
+ boolean exist) {
+ for (LocatedBlock locatedBlock : lbs.getLocatedBlocks()) {
+ if (exist) {
+ assertTrue(bm.getStoredBlock(locatedBlock.getBlock().
+ getLocalBlock()) != null);
+ } else {
+ assertTrue(bm.getStoredBlock(locatedBlock.getBlock().
+ getLocalBlock()) == null);
+ }
+ }
+ }
+
+ private byte[] readAll(FSDataInputStream in) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int n = 0;
+ while((n = in.read(buffer)) > -1) {
+ out.write(buffer, 0, n);
+ }
+ return out.toByteArray();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index a5e2edf..3f96c0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -99,7 +99,7 @@ public class CreateEditsLog {
INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
fileUc.toUnderConstruction("", "");
- editLog.logOpenFile(filePath, fileUc, false);
+ editLog.logOpenFile(filePath, fileUc, false, false);
editLog.logCloseFile(filePath, inode);
if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 47a807a..7b62242 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -197,7 +197,7 @@ public class TestEditLog {
p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
inode.toUnderConstruction("", "");
- editLog.logOpenFile("/filename" + (startIndex + i), inode, false);
+ editLog.logOpenFile("/filename" + (startIndex + i), inode, false, false);
editLog.logCloseFile("/filename" + (startIndex + i), inode);
editLog.logSync();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index a134969..5d93a50 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61045203/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 340667c..977be98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<EDITS>
- <EDITS_VERSION>-57</EDITS_VERSION>
+ <EDITS_VERSION>-58</EDITS_VERSION>
<RECORD>
<OPCODE>OP_START_LOG_SEGMENT</OPCODE>
<DATA>
@@ -42,6 +42,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -64,6 +65,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -174,6 +176,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -196,6 +199,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -204,17 +208,61 @@
</DATA>
</RECORD>
<RECORD>
- <OPCODE>OP_SET_REPLICATION</OPCODE>
+ <OPCODE>OP_ADD</OPCODE>
<DATA>
<TXID>17</TXID>
+ <LENGTH>0</LENGTH>
+ <INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
+ <MTIME>1402899229912</MTIME>
+ <ATIME>1402899229912</ATIME>
+ <BLOCKSIZE>512</BLOCKSIZE>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
+ <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>true</OVERWRITE>
+ <PERMISSION_STATUS>
+ <USERNAME>andrew</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <MODE>420</MODE>
+ </PERMISSION_STATUS>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>21</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
- <OPCODE>OP_SET_PERMISSIONS</OPCODE>
+ <OPCODE>OP_CLOSE</OPCODE>
<DATA>
<TXID>18</TXID>
+ <LENGTH>0</LENGTH>
+ <INODEID>0</INODEID>
+ <PATH>/file_create</PATH>
+ <REPLICATION>1</REPLICATION>
+ <MTIME>1402899229931</MTIME>
+ <ATIME>1402899229912</ATIME>
+ <BLOCKSIZE>512</BLOCKSIZE>
+ <CLIENT_NAME></CLIENT_NAME>
+ <CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
+ <PERMISSION_STATUS>
+ <USERNAME>andrew</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <MODE>420</MODE>
+ </PERMISSION_STATUS>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SET_REPLICATION</OPCODE>
+ <DATA>
+ <TXID>19</TXID>
+ <PATH>/file_create</PATH>
+ <REPLICATION>1</REPLICATION>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SET_PERMISSIONS</OPCODE>
+ <DATA>
+ <TXID>20</TXID>
<SRC>/file_create</SRC>
<MODE>511</MODE>
</DATA>
@@ -222,7 +270,7 @@
<RECORD>
<OPCODE>OP_SET_OWNER</OPCODE>
<DATA>
- <TXID>19</TXID>
+ <TXID>21</TXID>
<SRC>/file_create</SRC>
<USERNAME>newOwner</USERNAME>
</DATA>
@@ -230,7 +278,7 @@
<RECORD>
<OPCODE>OP_TIMES</OPCODE>
<DATA>
- <TXID>20</TXID>
+ <TXID>22</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_create</PATH>
<MTIME>1285195527000</MTIME>
@@ -240,7 +288,7 @@
<RECORD>
<OPCODE>OP_SET_QUOTA</OPCODE>
<DATA>
- <TXID>21</TXID>
+ <TXID>23</TXID>
<SRC>/directory_mkdir</SRC>
<NSQUOTA>1000</NSQUOTA>
<DSQUOTA>-1</DSQUOTA>
@@ -249,7 +297,7 @@
<RECORD>
<OPCODE>OP_RENAME</OPCODE>
<DATA>
- <TXID>22</TXID>
+ <TXID>24</TXID>
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
@@ -262,7 +310,7 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>23</TXID>
+ <TXID>25</TXID>
<LENGTH>0</LENGTH>
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
@@ -272,6 +320,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -284,21 +333,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>24</TXID>
+ <TXID>26</TXID>
<BLOCK_ID>1073741825</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>25</TXID>
+ <TXID>27</TXID>
<GENSTAMPV2>1001</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>26</TXID>
+ <TXID>28</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
<BLOCK_ID>1073741825</BLOCK_ID>
@@ -312,21 +361,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>27</TXID>
+ <TXID>29</TXID>
<BLOCK_ID>1073741826</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>28</TXID>
+ <TXID>30</TXID>
<GENSTAMPV2>1002</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>29</TXID>
+ <TXID>31</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
<BLOCK_ID>1073741825</BLOCK_ID>
@@ -345,21 +394,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>30</TXID>
+ <TXID>32</TXID>
<BLOCK_ID>1073741827</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>31</TXID>
+ <TXID>33</TXID>
<GENSTAMPV2>1003</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>32</TXID>
+ <TXID>34</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
<BLOCK_ID>1073741826</BLOCK_ID>
@@ -378,7 +427,7 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>33</TXID>
+ <TXID>35</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
@@ -388,6 +437,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<BLOCK>
<BLOCK_ID>1073741825</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
@@ -413,7 +463,7 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>34</TXID>
+ <TXID>36</TXID>
<LENGTH>0</LENGTH>
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
@@ -423,6 +473,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -435,21 +486,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>35</TXID>
+ <TXID>37</TXID>
<BLOCK_ID>1073741828</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>36</TXID>
+ <TXID>38</TXID>
<GENSTAMPV2>1004</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>37</TXID>
+ <TXID>39</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
<BLOCK_ID>1073741828</BLOCK_ID>
@@ -463,21 +514,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>38</TXID>
+ <TXID>40</TXID>
<BLOCK_ID>1073741829</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>39</TXID>
+ <TXID>41</TXID>
<GENSTAMPV2>1005</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>40</TXID>
+ <TXID>42</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
<BLOCK_ID>1073741828</BLOCK_ID>
@@ -496,21 +547,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>41</TXID>
+ <TXID>43</TXID>
<BLOCK_ID>1073741830</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>42</TXID>
+ <TXID>44</TXID>
<GENSTAMPV2>1006</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>43</TXID>
+ <TXID>45</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
<BLOCK_ID>1073741829</BLOCK_ID>
@@ -529,7 +580,7 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>44</TXID>
+ <TXID>46</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
@@ -539,6 +590,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<BLOCK>
<BLOCK_ID>1073741828</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
@@ -564,7 +616,7 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>45</TXID>
+ <TXID>47</TXID>
<LENGTH>0</LENGTH>
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
@@ -574,6 +626,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -586,21 +639,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>46</TXID>
+ <TXID>48</TXID>
<BLOCK_ID>1073741831</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>47</TXID>
+ <TXID>49</TXID>
<GENSTAMPV2>1007</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>48</TXID>
+ <TXID>50</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
<BLOCK_ID>1073741831</BLOCK_ID>
@@ -614,21 +667,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>49</TXID>
+ <TXID>51</TXID>
<BLOCK_ID>1073741832</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>50</TXID>
+ <TXID>52</TXID>
<GENSTAMPV2>1008</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>51</TXID>
+ <TXID>53</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
<BLOCK_ID>1073741831</BLOCK_ID>
@@ -647,21 +700,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>52</TXID>
+ <TXID>54</TXID>
<BLOCK_ID>1073741833</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>53</TXID>
+ <TXID>55</TXID>
<GENSTAMPV2>1009</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>54</TXID>
+ <TXID>56</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
<BLOCK_ID>1073741832</BLOCK_ID>
@@ -680,7 +733,7 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>55</TXID>
+ <TXID>57</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
@@ -690,6 +743,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<BLOCK>
<BLOCK_ID>1073741831</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
@@ -715,7 +769,7 @@
<RECORD>
<OPCODE>OP_CONCAT_DELETE</OPCODE>
<DATA>
- <TXID>56</TXID>
+ <TXID>58</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
<TIMESTAMP>1402899230394</TIMESTAMP>
@@ -730,7 +784,7 @@
<RECORD>
<OPCODE>OP_SYMLINK</OPCODE>
<DATA>
- <TXID>57</TXID>
+ <TXID>59</TXID>
<LENGTH>0</LENGTH>
<INODEID>16392</INODEID>
<PATH>/file_symlink</PATH>
@@ -749,7 +803,7 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>58</TXID>
+ <TXID>60</TXID>
<LENGTH>0</LENGTH>
<INODEID>16393</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
@@ -759,6 +813,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
@@ -771,21 +826,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>59</TXID>
+ <TXID>61</TXID>
<BLOCK_ID>1073741834</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>60</TXID>
+ <TXID>62</TXID>
<GENSTAMPV2>1010</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>61</TXID>
+ <TXID>63</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
<BLOCK_ID>1073741834</BLOCK_ID>
@@ -799,7 +854,7 @@
<RECORD>
<OPCODE>OP_UPDATE_BLOCKS</OPCODE>
<DATA>
- <TXID>62</TXID>
+ <TXID>64</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
<BLOCK_ID>1073741834</BLOCK_ID>
@@ -813,14 +868,14 @@
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>63</TXID>
+ <TXID>65</TXID>
<GENSTAMPV2>1011</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
- <TXID>64</TXID>
+ <TXID>66</TXID>
<LEASEHOLDER>DFSClient_NONMAPREDUCE_1233039831_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
@@ -829,7 +884,7 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>65</TXID>
+ <TXID>67</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
@@ -839,6 +894,7 @@
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
<BLOCK>
<BLOCK_ID>1073741834</BLOCK_ID>
<NUM_BYTES>11</NUM_BYTES>
@@ -854,7 +910,7 @@
<RECORD>
<OPCODE>OP_ADD_CACHE_POOL</OPCODE>
<DATA>
- <TXID>66</TXID>
+ <TXID>68</TXID>
<POOLNAME>pool1</POOLNAME>
<OWNERNAME>andrew</OWNERNAME>
<GROUPNAME>andrew</GROUPNAME>
@@ -868,7 +924,7 @@
<RECORD>
<OPCODE>OP_MODIFY_CACHE_POOL</OPCODE>
<DATA>
- <TXID>67</TXID>
+ <TXID>69</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
@@ -878,7 +934,7 @@
<RECORD>
<OPCODE>OP_ADD_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>68</TXID>
+ <TXID>70</TXID>
<ID>1</ID>
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
@@ -891,7 +947,7 @@
<RECORD>
<OPCODE>OP_MODIFY_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>69</TXID>
+ <TXID>71</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
@@ -901,7 +957,7 @@
<RECORD>
<OPCODE>OP_REMOVE_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>70</TXID>
+ <TXID>72</TXID>
<ID>1</ID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>77</RPC_CALLID>
@@ -910,7 +966,7 @@
<RECORD>
<OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
<DATA>
- <TXID>71</TXID>
+ <TXID>73</TXID>
<POOLNAME>pool1</POOLNAME>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>78</RPC_CALLID>
@@ -919,7 +975,7 @@
<RECORD>
<OPCODE>OP_SET_ACL</OPCODE>
<DATA>
- <TXID>72</TXID>
+ <TXID>74</TXID>
<SRC>/file_concat_target</SRC>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
@@ -952,7 +1008,7 @@
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>73</TXID>
+ <TXID>75</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
@@ -966,7 +1022,7 @@
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>74</TXID>
+ <TXID>76</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
@@ -980,7 +1036,7 @@
<RECORD>
<OPCODE>OP_REMOVE_XATTR</OPCODE>
<DATA>
- <TXID>75</TXID>
+ <TXID>77</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
@@ -993,21 +1049,21 @@
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
- <TXID>76</TXID>
+ <TXID>78</TXID>
<STARTTIME>1402899233646</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
- <TXID>77</TXID>
+ <TXID>79</TXID>
<FINALIZETIME>1402899233647</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_END_LOG_SEGMENT</OPCODE>
<DATA>
- <TXID>78</TXID>
+ <TXID>80</TXID>
</DATA>
</RECORD>
</EDITS>