You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2014/06/22 00:59:34 UTC
svn commit: r1604488 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolPB/
src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/a...
Author: jing9
Date: Sat Jun 21 22:59:34 2014
New Revision: 1604488
URL: http://svn.apache.org/r1604488
Log:
HDFS-4667. Capture renamed files/directories in snapshot diff report. Contributed by Jing Zhao and Binglin Chang.
Removed:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sat Jun 21 22:59:34 2014
@@ -458,6 +458,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9)
+ HDFS-4667. Capture renamed files/directories in snapshot diff report. (jing9
+ and Binglin Chang via jing9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java Sat Jun 21 22:59:34 2014
@@ -25,6 +25,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
+import com.google.common.base.Objects;
+
/**
* This class represents to end users the difference between two snapshots of
* the same directory, or the difference between a snapshot of the directory and
@@ -79,43 +81,64 @@ public class SnapshotDiffReport {
/** The type of the difference. */
private final DiffType type;
/**
- * The relative path (related to the snapshot root) of the file/directory
- * where changes have happened
+ * The relative path (related to the snapshot root) of 1) the file/directory
+ * where changes have happened, or 2) the source file/dir of a rename op.
*/
- private final byte[] relativePath;
+ private final byte[] sourcePath;
+ private final byte[] targetPath;
+
+ public DiffReportEntry(DiffType type, byte[] sourcePath) {
+ this(type, sourcePath, null);
+ }
+
+ public DiffReportEntry(DiffType type, byte[][] sourcePathComponents) {
+ this(type, sourcePathComponents, null);
+ }
- public DiffReportEntry(DiffType type, byte[] path) {
+ public DiffReportEntry(DiffType type, byte[] sourcePath, byte[] targetPath) {
this.type = type;
- this.relativePath = path;
+ this.sourcePath = sourcePath;
+ this.targetPath = targetPath;
}
- public DiffReportEntry(DiffType type, byte[][] pathComponents) {
+ public DiffReportEntry(DiffType type, byte[][] sourcePathComponents,
+ byte[][] targetPathComponents) {
this.type = type;
- this.relativePath = DFSUtil.byteArray2bytes(pathComponents);
+ this.sourcePath = DFSUtil.byteArray2bytes(sourcePathComponents);
+ this.targetPath = targetPathComponents == null ? null : DFSUtil
+ .byteArray2bytes(targetPathComponents);
}
@Override
public String toString() {
- return type.getLabel() + "\t" + getRelativePathString();
+ String str = type.getLabel() + "\t" + getPathString(sourcePath);
+ if (type == DiffType.RENAME) {
+ str += " -> " + getPathString(targetPath);
+ }
+ return str;
}
public DiffType getType() {
return type;
}
- public String getRelativePathString() {
- String path = DFSUtil.bytes2String(relativePath);
- if (path.isEmpty()) {
+ static String getPathString(byte[] path) {
+ String pathStr = DFSUtil.bytes2String(path);
+ if (pathStr.isEmpty()) {
return Path.CUR_DIR;
} else {
- return Path.CUR_DIR + Path.SEPARATOR + path;
+ return Path.CUR_DIR + Path.SEPARATOR + pathStr;
}
}
- public byte[] getRelativePath() {
- return relativePath;
+ public byte[] getSourcePath() {
+ return sourcePath;
}
-
+
+ public byte[] getTargetPath() {
+ return targetPath;
+ }
+
@Override
public boolean equals(Object other) {
if (this == other) {
@@ -124,14 +147,15 @@ public class SnapshotDiffReport {
if (other != null && other instanceof DiffReportEntry) {
DiffReportEntry entry = (DiffReportEntry) other;
return type.equals(entry.getType())
- && Arrays.equals(relativePath, entry.getRelativePath());
+ && Arrays.equals(sourcePath, entry.getSourcePath())
+ && Arrays.equals(targetPath, entry.getTargetPath());
}
return false;
}
@Override
public int hashCode() {
- return Arrays.hashCode(relativePath);
+ return Objects.hashCode(getSourcePath(), getTargetPath());
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Sat Jun 21 22:59:34 2014
@@ -44,7 +44,6 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.StorageType;
-import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -1737,24 +1736,29 @@ public class PBHelper {
}
DiffType type = DiffType.getTypeFromLabel(entry
.getModificationLabel());
- return type == null ? null :
- new DiffReportEntry(type, entry.getFullpath().toByteArray());
+ return type == null ? null : new DiffReportEntry(type, entry.getFullpath()
+ .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath()
+ .toByteArray() : null);
}
public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) {
if (entry == null) {
return null;
}
- byte[] fullPath = entry.getRelativePath();
- ByteString fullPathString = ByteString
- .copyFrom(fullPath == null ? DFSUtil.EMPTY_BYTES : fullPath);
-
+ ByteString sourcePath = ByteString
+ .copyFrom(entry.getSourcePath() == null ? DFSUtil.EMPTY_BYTES : entry
+ .getSourcePath());
String modification = entry.getType().getLabel();
-
- SnapshotDiffReportEntryProto entryProto = SnapshotDiffReportEntryProto
- .newBuilder().setFullpath(fullPathString)
- .setModificationLabel(modification).build();
- return entryProto;
+ SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
+ .newBuilder().setFullpath(sourcePath)
+ .setModificationLabel(modification);
+ if (entry.getType() == DiffType.RENAME) {
+ ByteString targetPath = ByteString
+ .copyFrom(entry.getTargetPath() == null ? DFSUtil.EMPTY_BYTES : entry
+ .getTargetPath());
+ builder.setTargetPath(targetPath);
+ }
+ return builder.build();
}
public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Sat Jun 21 22:59:34 2014
@@ -2790,7 +2790,7 @@ public class FSDirectory implements Clos
* Note that this method cannot handle scenarios where the inode is in a
* snapshot.
*/
- static byte[][] getPathComponents(INode inode) {
+ public static byte[][] getPathComponents(INode inode) {
List<byte[]> components = new ArrayList<byte[]>();
components.add(0, inode.getLocalNameBytes());
while(inode.getParent() != null) {
@@ -2801,53 +2801,6 @@ public class FSDirectory implements Clos
}
/**
- * The same functionality with {@link #getPathComponents(INode)}, but can
- * handle snapshots.
- */
- public static byte[][] getPathComponentsWithSnapshot(INode inode) {
- List<byte[]> components = new ArrayList<byte[]>();
- boolean inSnapshot = false;
- int snapshotId = Snapshot.CURRENT_STATE_ID;
- do {
- if (inode instanceof INodeReference.WithCount) {
- // identify the corresponding WithName or DstReference node
- inode = ((WithCount) inode).getParentRef(snapshotId);
- } else { // normal INode and WithName/DstReference
- if (inode instanceof INodeDirectory
- && inode.asDirectory().isSnapshottable() && inSnapshot
- && snapshotId != Snapshot.CURRENT_STATE_ID) {
- INodeDirectorySnapshottable sdir = (INodeDirectorySnapshottable) inode
- .asDirectory();
- Snapshot snapshot = sdir.getSnapshotById(snapshotId);
- if (snapshot != null) {
- components.add(0, snapshot.getRoot().getLocalNameBytes());
- components.add(0, HdfsConstants.DOT_SNAPSHOT_DIR_BYTES);
- // the snapshot has been found, thus no need to check snapshottable
- // directory afterwards
- inSnapshot = false;
- }
- }
- INode parent = inode.getParentReference() != null ? inode
- .getParentReference() : inode.getParent();
- if (parent != null && parent instanceof INodeDirectory) {
- int sid = parent.asDirectory().searchChild(inode);
- Preconditions.checkState(sid != Snapshot.NO_SNAPSHOT_ID);
- if (sid != Snapshot.CURRENT_STATE_ID
- && snapshotId == Snapshot.CURRENT_STATE_ID) {
- snapshotId = sid;
- inSnapshot = true;
- }
- components.add(0, inode.getLocalNameBytes());
- } else if (parent == null) { // root
- components.add(0, inode.getLocalNameBytes());
- }
- inode = parent;
- }
- } while (inode != null);
- return components.toArray(new byte[components.size()][]);
- }
-
- /**
* @return path components for reserved path, else null.
*/
static byte[][] getPathComponentsForReservedPath(String src) {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Sat Jun 21 22:59:34 2014
@@ -26,11 +26,9 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
-import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
@@ -365,7 +363,7 @@ public class INodeDirectory extends INod
* children list nor in any snapshot; otherwise the snapshot id of the
* corresponding snapshot diff list.
*/
- int searchChild(INode inode) {
+ public int searchChild(INode inode) {
INode child = getChild(inode.getLocalNameBytes(), Snapshot.CURRENT_STATE_ID);
if (child != inode) {
// inode is not in parent's children list, thus inode must be in
@@ -764,7 +762,9 @@ public class INodeDirectory extends INod
public boolean metadataEquals(INodeDirectoryAttributes other) {
return other != null
&& getQuotaCounts().equals(other.getQuotaCounts())
- && getPermissionLong() == other.getPermissionLong();
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
}
/*
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java Sat Jun 21 22:59:34 2014
@@ -53,8 +53,10 @@ public interface INodeDirectoryAttribute
@Override
public boolean metadataEquals(INodeDirectoryAttributes other) {
return other != null
- && this.getQuotaCounts().equals(other.getQuotaCounts())
- && getPermissionLong() == other.getPermissionLong();
+ && getQuotaCounts().equals(other.getQuotaCounts())
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Sat Jun 21 22:59:34 2014
@@ -144,6 +144,15 @@ public class INodeFile extends INodeWith
return this;
}
+ @Override
+ public boolean metadataEquals(INodeFileAttributes other) {
+ return other != null
+ && getHeaderLong()== other.getHeaderLong()
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
+ }
+
/* Start of Under-Construction Feature */
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java Sat Jun 21 22:59:34 2014
@@ -36,6 +36,8 @@ public interface INodeFileAttributes ext
/** @return the header as a long. */
public long getHeaderLong();
+ public boolean metadataEquals(INodeFileAttributes other);
+
/** A copy of the inode file attributes */
public static class SnapshotCopy extends INodeAttributes.SnapshotCopy
implements INodeFileAttributes {
@@ -70,5 +72,14 @@ public interface INodeFileAttributes ext
public long getHeaderLong() {
return header;
}
+
+ @Override
+ public boolean metadataEquals(INodeFileAttributes other) {
+ return other != null
+ && getHeaderLong()== other.getHeaderLong()
+ && getPermissionLong() == other.getPermissionLong()
+ && getAclFeature() == other.getAclFeature()
+ && getXAttrFeature() == other.getXAttrFeature();
+ }
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java Sat Jun 21 22:59:34 2014
@@ -435,21 +435,28 @@ public abstract class INodeReference ext
}
}
+ /**
+ * @return the WithName/DstReference node contained in the given snapshot.
+ */
public INodeReference getParentRef(int snapshotId) {
- // when the given snapshotId is CURRENT_STATE_ID, it is possible that we
- // do not know where the corresponding inode belongs, thus we simply
- // return the last reference node
- if (snapshotId == Snapshot.CURRENT_STATE_ID) {
- return this.getParentReference() != null ? this.getParentReference()
- : this.getLastWithName();
- }
- // otherwise we search the withNameList
- for (int i = 0; i < withNameList.size(); i++) {
- if (snapshotId <= withNameList.get(i).lastSnapshotId) {
- return withNameList.get(i);
+ int start = 0;
+ int end = withNameList.size() - 1;
+ while (start < end) {
+ int mid = start + (end - start) / 2;
+ int sid = withNameList.get(mid).lastSnapshotId;
+ if (sid == snapshotId) {
+ return withNameList.get(mid);
+ } else if (sid < snapshotId) {
+ start = mid + 1;
+ } else {
+ end = mid;
}
}
- return this.getParentReference();
+ if (withNameList.get(start).lastSnapshotId >= snapshotId) {
+ return withNameList.get(start);
+ } else {
+ return this.getParentReference();
+ }
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java Sat Jun 21 22:59:34 2014
@@ -227,32 +227,34 @@ abstract class AbstractINodeDiffList<N e
D diff = getDiffById(snapshotId);
return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId();
}
-
- /**
- * Check if changes have happened between two snapshots.
- * @param earlier The snapshot taken earlier
- * @param later The snapshot taken later
- * @return Whether or not modifications (including diretory/file metadata
- * change, file creation/deletion under the directory) have happened
- * between snapshots.
- */
- final boolean changedBetweenSnapshots(Snapshot earlier, Snapshot later) {
+
+ final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) {
+ Snapshot earlier = from;
+ Snapshot later = to;
+ if (Snapshot.ID_COMPARATOR.compare(from, to) > 0) {
+ earlier = to;
+ later = from;
+ }
+
final int size = diffs.size();
int earlierDiffIndex = Collections.binarySearch(diffs, earlier.getId());
+ int laterDiffIndex = later == null ? size : Collections
+ .binarySearch(diffs, later.getId());
if (-earlierDiffIndex - 1 == size) {
// if the earlierSnapshot is after the latest SnapshotDiff stored in
// diffs, no modification happened after the earlierSnapshot
- return false;
+ return null;
}
- if (later != null) {
- int laterDiffIndex = Collections.binarySearch(diffs, later.getId());
- if (laterDiffIndex == -1 || laterDiffIndex == 0) {
- // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or
- // before it, no modification happened before the laterSnapshot
- return false;
- }
+ if (laterDiffIndex == -1 || laterDiffIndex == 0) {
+ // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or
+ // before it, no modification happened before the laterSnapshot
+ return null;
}
- return true;
+ earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
+ : earlierDiffIndex;
+ laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
+ : laterDiffIndex;
+ return new int[]{earlierDiffIndex, laterDiffIndex};
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java Sat Jun 21 22:59:34 2014
@@ -21,7 +21,6 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
@@ -42,6 +41,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.Diff;
import org.apache.hadoop.hdfs.util.Diff.Container;
@@ -170,49 +170,33 @@ public class DirectoryWithSnapshotFeatur
* @return A list of {@link DiffReportEntry} as the diff report.
*/
public List<DiffReportEntry> generateReport(byte[][] parentPath,
- boolean fromEarlier) {
- List<DiffReportEntry> cList = new ArrayList<DiffReportEntry>();
- List<DiffReportEntry> dList = new ArrayList<DiffReportEntry>();
- int c = 0, d = 0;
+ boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
+ List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
List<INode> created = getList(ListType.CREATED);
List<INode> deleted = getList(ListType.DELETED);
byte[][] fullPath = new byte[parentPath.length + 1][];
System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
- for (; c < created.size() && d < deleted.size(); ) {
- INode cnode = created.get(c);
- INode dnode = deleted.get(d);
- if (cnode.compareTo(dnode.getLocalNameBytes()) == 0) {
+ for (INode cnode : created) {
+ RenameEntry entry = renameMap.get(cnode.getId());
+ if (entry == null || !entry.isRename()) {
fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
- // must be the case: delete first and then create an inode with the
- // same name
- cList.add(new DiffReportEntry(DiffType.CREATE, fullPath));
- dList.add(new DiffReportEntry(DiffType.DELETE, fullPath));
- c++;
- d++;
- } else if (cnode.compareTo(dnode.getLocalNameBytes()) < 0) {
- fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
- cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
+ list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
: DiffType.DELETE, fullPath));
- c++;
+ }
+ }
+ for (INode dnode : deleted) {
+ RenameEntry entry = renameMap.get(dnode.getId());
+ if (entry != null && entry.isRename()) {
+ list.add(new DiffReportEntry(DiffType.RENAME,
+ fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
+ fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
} else {
fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
- dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
+ list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
: DiffType.CREATE, fullPath));
- d++;
}
}
- for (; d < deleted.size(); d++) {
- fullPath[fullPath.length - 1] = deleted.get(d).getLocalNameBytes();
- dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
- : DiffType.CREATE, fullPath));
- }
- for (; c < created.size(); c++) {
- fullPath[fullPath.length - 1] = created.get(c).getLocalNameBytes();
- cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
- : DiffType.DELETE, fullPath));
- }
- dList.addAll(cList);
- return dList;
+ return list;
}
}
@@ -724,34 +708,21 @@ public class DirectoryWithSnapshotFeatur
*/
boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot,
Snapshot toSnapshot, ChildrenDiff diff, INodeDirectory currentINode) {
- Snapshot earlier = fromSnapshot;
- Snapshot later = toSnapshot;
- if (Snapshot.ID_COMPARATOR.compare(fromSnapshot, toSnapshot) > 0) {
- earlier = toSnapshot;
- later = fromSnapshot;
- }
-
- boolean modified = diffs.changedBetweenSnapshots(earlier, later);
- if (!modified) {
+ int[] diffIndexPair = diffs.changedBetweenSnapshots(fromSnapshot,
+ toSnapshot);
+ if (diffIndexPair == null) {
return false;
}
-
- final List<DirectoryDiff> difflist = diffs.asList();
- final int size = difflist.size();
- int earlierDiffIndex = Collections.binarySearch(difflist, earlier.getId());
- int laterDiffIndex = later == null ? size : Collections
- .binarySearch(difflist, later.getId());
- earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
- : earlierDiffIndex;
- laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
- : laterDiffIndex;
+ int earlierDiffIndex = diffIndexPair[0];
+ int laterDiffIndex = diffIndexPair[1];
boolean dirMetadataChanged = false;
INodeDirectoryAttributes dirCopy = null;
+ List<DirectoryDiff> difflist = diffs.asList();
for (int i = earlierDiffIndex; i < laterDiffIndex; i++) {
DirectoryDiff sdiff = difflist.get(i);
diff.combinePosterior(sdiff.diff, null);
- if (dirMetadataChanged == false && sdiff.snapshotINode != null) {
+ if (!dirMetadataChanged && sdiff.snapshotINode != null) {
if (dirCopy == null) {
dirCopy = sdiff.snapshotINode;
} else if (!dirCopy.metadataEquals(sdiff.snapshotINode)) {
@@ -763,7 +734,7 @@ public class DirectoryWithSnapshotFeatur
if (!diff.isEmpty() || dirMetadataChanged) {
return true;
} else if (dirCopy != null) {
- for (int i = laterDiffIndex; i < size; i++) {
+ for (int i = laterDiffIndex; i < difflist.size(); i++) {
if (!dirCopy.metadataEquals(difflist.get(i).snapshotINode)) {
return true;
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java Sat Jun 21 22:59:34 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
import org.apache.hadoop.hdfs.server.namenode.Quota;
/**
@@ -73,7 +74,41 @@ public class FileWithSnapshotFeature imp
}
return max;
}
-
+
+ boolean changedBetweenSnapshots(INodeFile file, Snapshot from, Snapshot to) {
+ int[] diffIndexPair = diffs.changedBetweenSnapshots(from, to);
+ if (diffIndexPair == null) {
+ return false;
+ }
+ int earlierDiffIndex = diffIndexPair[0];
+ int laterDiffIndex = diffIndexPair[1];
+
+ final List<FileDiff> diffList = diffs.asList();
+ final long earlierLength = diffList.get(earlierDiffIndex).getFileSize();
+ final long laterLength = laterDiffIndex == diffList.size() ? file
+ .computeFileSize(true, false) : diffList.get(laterDiffIndex)
+ .getFileSize();
+ if (earlierLength != laterLength) { // file length has been changed
+ return true;
+ }
+
+ INodeFileAttributes earlierAttr = null; // check the metadata
+ for (int i = earlierDiffIndex; i < laterDiffIndex; i++) {
+ FileDiff diff = diffList.get(i);
+ if (diff.snapshotINode != null) {
+ earlierAttr = diff.snapshotINode;
+ break;
+ }
+ }
+ if (earlierAttr == null) { // no meta-change at all, return false
+ return false;
+ }
+ INodeFileAttributes laterAttr = diffs.getSnapshotINode(
+ Math.max(Snapshot.getSnapshotId(from), Snapshot.getSnapshotId(to)),
+ file);
+ return !earlierAttr.metadataEquals(laterAttr);
+ }
+
public String getDetailedString() {
return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs;
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Sat Jun 21 22:59:34 2014
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
@@ -43,6 +44,9 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeMap;
+import org.apache.hadoop.hdfs.server.namenode.INodeReference;
+import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
+import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
import org.apache.hadoop.hdfs.server.namenode.Quota;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
@@ -51,6 +55,7 @@ import org.apache.hadoop.hdfs.util.ReadO
import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import com.google.common.primitives.SignedBytes;
/**
@@ -98,7 +103,43 @@ public class INodeDirectorySnapshottable
}
}
};
-
+
+ static class RenameEntry {
+ private byte[][] sourcePath;
+ private byte[][] targetPath;
+
+ void setSource(INode source, byte[][] sourceParentPath) {
+ Preconditions.checkState(sourcePath == null);
+ sourcePath = new byte[sourceParentPath.length + 1][];
+ System.arraycopy(sourceParentPath, 0, sourcePath, 0,
+ sourceParentPath.length);
+ sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
+ }
+
+ void setTarget(INode target, byte[][] targetParentPath) {
+ targetPath = new byte[targetParentPath.length + 1][];
+ System.arraycopy(targetParentPath, 0, targetPath, 0,
+ targetParentPath.length);
+ targetPath[targetPath.length - 1] = target.getLocalNameBytes();
+ }
+
+ void setTarget(byte[][] targetPath) {
+ this.targetPath = targetPath;
+ }
+
+ boolean isRename() {
+ return sourcePath != null && targetPath != null;
+ }
+
+ byte[][] getSourcePath() {
+ return sourcePath;
+ }
+
+ byte[][] getTargetPath() {
+ return targetPath;
+ }
+ }
+
/** The root directory of the snapshots */
private final INodeDirectorySnapshottable snapshotRoot;
/** The starting point of the difference */
@@ -109,7 +150,7 @@ public class INodeDirectorySnapshottable
* A map recording modified INodeFile and INodeDirectory and their relative
* path corresponding to the snapshot root. Sorted based on their names.
*/
- private final SortedMap<INode, byte[][]> diffMap =
+ private final SortedMap<INode, byte[][]> diffMap =
new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
/**
* A map capturing the detailed difference about file creation/deletion.
@@ -119,7 +160,10 @@ public class INodeDirectorySnapshottable
*/
private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
new HashMap<INodeDirectory, ChildrenDiff>();
-
+
+ private final Map<Long, RenameEntry> renameMap =
+ new HashMap<Long, RenameEntry>();
+
SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
Snapshot end) {
this.snapshotRoot = snapshotRoot;
@@ -132,8 +176,36 @@ public class INodeDirectorySnapshottable
ChildrenDiff diff) {
dirDiffMap.put(dir, diff);
diffMap.put(dir, relativePath);
+ // detect rename
+ for (INode created : diff.getList(ListType.CREATED)) {
+ if (created.isReference()) {
+ RenameEntry entry = getEntry(created.getId());
+ if (entry.getTargetPath() == null) {
+ entry.setTarget(created, relativePath);
+ }
+ }
+ }
+ for (INode deleted : diff.getList(ListType.DELETED)) {
+ if (deleted instanceof INodeReference.WithName) {
+ RenameEntry entry = getEntry(deleted.getId());
+ entry.setSource(deleted, relativePath);
+ }
+ }
}
-
+
+ private RenameEntry getEntry(long inodeId) {
+ RenameEntry entry = renameMap.get(inodeId);
+ if (entry == null) {
+ entry = new RenameEntry();
+ renameMap.put(inodeId, entry);
+ }
+ return entry;
+ }
+
+ private void setRenameTarget(long inodeId, byte[][] path) {
+ getEntry(inodeId).setTarget(path);
+ }
+
/** Add a modified file */
private void addFileDiff(INodeFile file, byte[][] relativePath) {
diffMap.put(file, relativePath);
@@ -152,11 +224,11 @@ public class INodeDirectorySnapshottable
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
for (INode node : diffMap.keySet()) {
diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
- .get(node)));
+ .get(node), null));
if (node.isDirectory()) {
ChildrenDiff dirDiff = dirDiffMap.get(node);
List<DiffReportEntry> subList = dirDiff.generateReport(
- diffMap.get(node), isFromEarlier());
+ diffMap.get(node), isFromEarlier(), renameMap);
diffReportList.addAll(subList);
}
}
@@ -423,25 +495,37 @@ public class INodeDirectorySnapshottable
*/
private void computeDiffRecursively(INode node, List<byte[]> parentPath,
SnapshotDiffInfo diffReport) {
- ChildrenDiff diff = new ChildrenDiff();
+ final Snapshot earlierSnapshot = diffReport.isFromEarlier() ?
+ diffReport.from : diffReport.to;
+ final Snapshot laterSnapshot = diffReport.isFromEarlier() ?
+ diffReport.to : diffReport.from;
byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
if (node.isDirectory()) {
+ final ChildrenDiff diff = new ChildrenDiff();
INodeDirectory dir = node.asDirectory();
DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
if (sf != null) {
- boolean change = sf.computeDiffBetweenSnapshots(diffReport.from,
- diffReport.to, diff, dir);
+ boolean change = sf.computeDiffBetweenSnapshots(earlierSnapshot,
+ laterSnapshot, diff, dir);
if (change) {
diffReport.addDirDiff(dir, relativePath, diff);
}
}
- ReadOnlyList<INode> children = dir.getChildrenList(
- diffReport.isFromEarlier() ? Snapshot.getSnapshotId(diffReport.to) :
- Snapshot.getSnapshotId(diffReport.from));
+ ReadOnlyList<INode> children = dir.getChildrenList(earlierSnapshot
+ .getId());
for (INode child : children) {
final byte[] name = child.getLocalNameBytes();
- if (diff.searchIndex(ListType.CREATED, name) < 0
- && diff.searchIndex(ListType.DELETED, name) < 0) {
+ boolean toProcess = diff.searchIndex(ListType.DELETED, name) < 0;
+ if (!toProcess && child instanceof INodeReference.WithName) {
+ byte[][] renameTargetPath = findRenameTargetPath((WithName) child,
+ laterSnapshot == null ? Snapshot.CURRENT_STATE_ID :
+ laterSnapshot.getId());
+ if (renameTargetPath != null) {
+ toProcess = true;
+ diffReport.setRenameTarget(child.getId(), renameTargetPath);
+ }
+ }
+ if (toProcess) {
parentPath.add(name);
computeDiffRecursively(child, parentPath, diffReport);
parentPath.remove(parentPath.size() - 1);
@@ -449,18 +533,47 @@ public class INodeDirectorySnapshottable
}
} else if (node.isFile() && node.asFile().isWithSnapshot()) {
INodeFile file = node.asFile();
- Snapshot earlierSnapshot = diffReport.isFromEarlier() ? diffReport.from
- : diffReport.to;
- Snapshot laterSnapshot = diffReport.isFromEarlier() ? diffReport.to
- : diffReport.from;
- boolean change = file.getDiffs().changedBetweenSnapshots(earlierSnapshot,
- laterSnapshot);
+ boolean change = file.getFileWithSnapshotFeature()
+ .changedBetweenSnapshots(file, earlierSnapshot, laterSnapshot);
if (change) {
diffReport.addFileDiff(file, relativePath);
}
}
}
-
+
+ /**
+ * We just found a deleted WithName node as the source of a rename operation.
+ * However, we should include it in our snapshot diff report as rename only
+ * if the rename target is also under the same snapshottable directory.
+ */
+ private byte[][] findRenameTargetPath(INodeReference.WithName wn,
+ final int snapshotId) {
+ INode inode = wn.getReferredINode();
+ final LinkedList<byte[]> ancestors = Lists.newLinkedList();
+ while (inode != null) {
+ if (inode == this) {
+ return ancestors.toArray(new byte[ancestors.size()][]);
+ }
+ if (inode instanceof INodeReference.WithCount) {
+ inode = ((WithCount) inode).getParentRef(snapshotId);
+ } else {
+ INode parent = inode.getParentReference() != null ? inode
+ .getParentReference() : inode.getParent();
+ if (parent != null && parent instanceof INodeDirectory) {
+ int sid = parent.asDirectory().searchChild(inode);
+ if (sid < snapshotId) {
+ return null;
+ }
+ }
+ if (!(parent instanceof WithCount)) {
+ ancestors.addFirst(inode.getLocalNameBytes());
+ }
+ inode = parent;
+ }
+ }
+ return null;
+ }
+
/**
* Replace itself with {@link INodeDirectoryWithSnapshot} or
* {@link INodeDirectory} depending on the latest snapshot.
@@ -549,4 +662,4 @@ public class INodeDirectorySnapshottable
});
}
}
-}
+}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto Sat Jun 21 22:59:34 2014
@@ -275,6 +275,7 @@ message SnapshottableDirectoryListingPro
message SnapshotDiffReportEntryProto {
required bytes fullpath = 1;
required string modificationLabel = 2;
+ optional bytes targetPath = 3;
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml Sat Jun 21 22:59:34 2014
@@ -255,8 +255,36 @@
<tr><td>fromSnapshot</td><td>The name of the starting snapshot.</td></tr>
<tr><td>toSnapshot</td><td>The name of the ending snapshot.</td></tr>
</table></li>
+ <li>Results:
+ <table>
+ <tr><td>+</td><td>The file/directory has been created.</td></tr>
+ <tr><td>-</td><td>The file/directory has been deleted.</td></tr>
+ <tr><td>M</td><td>The file/directory has been modified.</td></tr>
+ <tr><td>R</td><td>The file/directory has been renamed.</td></tr>
+ </table>
+ </li>
</ul>
<p>
+ A <em>RENAME</em> entry indicates a file/directory has been renamed but
+ is still under the same snapshottable directory. A file/directory is
+ reported as deleted if it was renamed to outside of the snapshottble directory.
+ A file/directory renamed from outside of the snapshottble directory is
+ reported as newly created.
+ </p>
+ <p>
+ The snapshot difference report does not guarantee the same operation sequence.
+ For example, if we rename the directory <em>"/foo"</em> to <em>"/foo2"</em>, and
+ then append new data to the file <em>"/foo2/bar"</em>, the difference report will
+ be:
+ <source>
+ R. /foo -> /foo2
+ M. /foo/bar
+ </source>
+ I.e., the changes on the files/directories under a renamed directory is
+ reported using the original path before the rename (<em>"/foo/bar"</em> in
+ the above example).
+ </p>
+ <p>
See also the corresponding Java API
<code>SnapshotDiffReport getSnapshotDiffReport(Path path, String fromSnapshot, String toSnapshot)</code>
in <code>DistributedFileSystem</code>.
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java Sat Jun 21 22:59:34 2014
@@ -169,10 +169,11 @@ public class TestRenameWithSnapshots {
}
private static boolean existsInDiffReport(List<DiffReportEntry> entries,
- DiffType type, String relativePath) {
+ DiffType type, String sourcePath, String targetPath) {
for (DiffReportEntry entry : entries) {
- if ((entry.getType() == type)
- && ((new String(entry.getRelativePath())).compareTo(relativePath) == 0)) {
+ if (entry.equals(new DiffReportEntry(type, DFSUtil
+ .string2Bytes(sourcePath), targetPath == null ? null : DFSUtil
+ .string2Bytes(targetPath)))) {
return true;
}
}
@@ -195,8 +196,9 @@ public class TestRenameWithSnapshots {
SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, "");
List<DiffReportEntry> entries = diffReport.getDiffList();
assertTrue(entries.size() == 2);
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName()));
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(),
+ null));
}
/**
@@ -215,10 +217,10 @@ public class TestRenameWithSnapshots {
SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, "");
System.out.println("DiffList is " + diffReport.toString());
List<DiffReportEntry> entries = diffReport.getDiffList();
- assertTrue(entries.size() == 3);
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName()));
+ assertTrue(entries.size() == 2);
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(),
+ file2.getName()));
}
@Test (timeout=60000)
@@ -238,26 +240,26 @@ public class TestRenameWithSnapshots {
diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, snap2);
LOG.info("DiffList is " + diffReport.toString());
List<DiffReportEntry> entries = diffReport.getDiffList();
- assertTrue(entries.size() == 3);
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName()));
+ assertTrue(entries.size() == 2);
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(),
+ file2.getName()));
diffReport = hdfs.getSnapshotDiffReport(sub1, snap2, "");
LOG.info("DiffList is " + diffReport.toString());
entries = diffReport.getDiffList();
- assertTrue(entries.size() == 3);
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, file3.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, file2.getName()));
+ assertTrue(entries.size() == 2);
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, file2.getName(),
+ file3.getName()));
diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, "");
LOG.info("DiffList is " + diffReport.toString());
entries = diffReport.getDiffList();
- assertTrue(entries.size() == 3);
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, file3.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName()));
+ assertTrue(entries.size() == 2);
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(),
+ file3.getName()));
}
@Test (timeout=60000)
@@ -280,11 +282,10 @@ public class TestRenameWithSnapshots {
"");
LOG.info("DiffList is \n\"" + diffReport.toString() + "\"");
List<DiffReportEntry> entries = diffReport.getDiffList();
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, sub2.getName()
- + "/" + sub2file2.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, sub2.getName()
- + "/" + sub2file1.getName()));
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(),
+ null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName()
+ + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName()));
}
@Test (timeout=60000)
@@ -307,10 +308,10 @@ public class TestRenameWithSnapshots {
"");
LOG.info("DiffList is \n\"" + diffReport.toString() + "\"");
List<DiffReportEntry> entries = diffReport.getDiffList();
- assertEquals(3, entries.size());
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, sub3.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, sub2.getName()));
+ assertEquals(2, entries.size());
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(),
+ sub3.getName()));
}
/**
@@ -2406,12 +2407,12 @@ public class TestRenameWithSnapshots {
LOG.info("DiffList is \n\"" + report.toString() + "\"");
List<DiffReportEntry> entries = report.getDiffList();
assertEquals(7, entries.size());
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
- assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, bar.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.CREATE, newDir.getName()));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1"));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file2"));
- assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file3"));
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null));
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName(), null));
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, bar.getName(), null));
+ assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1", null));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, "bar", "newDir"));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file2", "newDir/file2"));
+ assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file3", "newDir/file1"));
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java?rev=1604488&r1=1604487&r2=1604488&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java Sat Jun 21 22:59:34 2014
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -143,7 +144,7 @@ public class TestSnapshotDiffReport {
hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir));
}
// modify file10
- hdfs.setReplication(file10, (short) (REPLICATION - 1));
+ hdfs.setReplication(file10, (short) (REPLICATION + 1));
}
/** check the correctness of the diff reports */
@@ -166,11 +167,11 @@ public class TestSnapshotDiffReport {
} else if (entry.getType() == DiffType.DELETE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
- new DiffReportEntry(DiffType.CREATE, entry.getRelativePath())));
+ new DiffReportEntry(DiffType.CREATE, entry.getSourcePath())));
} else if (entry.getType() == DiffType.CREATE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
- new DiffReportEntry(DiffType.DELETE, entry.getRelativePath())));
+ new DiffReportEntry(DiffType.DELETE, entry.getSourcePath())));
}
}
}
@@ -329,5 +330,166 @@ public class TestSnapshotDiffReport {
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("subsub1")));
}
-
+
+ /**
+ * Rename a directory to its prior descendant, and verify the diff report.
+ */
+ @Test
+ public void testDiffReportWithRename() throws Exception {
+ final Path root = new Path("/");
+ final Path sdir1 = new Path(root, "dir1");
+ final Path sdir2 = new Path(root, "dir2");
+ final Path foo = new Path(sdir1, "foo");
+ final Path bar = new Path(foo, "bar");
+ hdfs.mkdirs(bar);
+ hdfs.mkdirs(sdir2);
+
+ // create snapshot on root
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+
+ // /dir1/foo/bar -> /dir2/bar
+ final Path bar2 = new Path(sdir2, "bar");
+ hdfs.rename(bar, bar2);
+
+ // /dir1/foo -> /dir2/bar/foo
+ final Path foo2 = new Path(bar2, "foo");
+ hdfs.rename(foo, foo2);
+
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s2");
+ // let's delete /dir2 to make things more complicated
+ hdfs.delete(sdir2, true);
+
+ verifyDiffReport(root, "s1", "s2",
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
+ new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/foo"),
+ DFSUtil.string2Bytes("dir2/bar/foo")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
+ new DiffReportEntry(DiffType.MODIFY,
+ DFSUtil.string2Bytes("dir1/foo/bar")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")),
+ new DiffReportEntry(DiffType.RENAME, DFSUtil
+ .string2Bytes("dir1/foo/bar"), DFSUtil.string2Bytes("dir2/bar")));
+ }
+
+ /**
+ * Rename a file/dir outside of the snapshottable dir should be reported as
+ * deleted. Rename a file/dir from outside should be reported as created.
+ */
+ @Test
+ public void testDiffReportWithRenameOutside() throws Exception {
+ final Path root = new Path("/");
+ final Path dir1 = new Path(root, "dir1");
+ final Path dir2 = new Path(root, "dir2");
+ final Path foo = new Path(dir1, "foo");
+ final Path fileInFoo = new Path(foo, "file");
+ final Path bar = new Path(dir2, "bar");
+ final Path fileInBar = new Path(bar, "file");
+ DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed);
+ DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, seed);
+
+ // create snapshot on /dir1
+ SnapshotTestHelper.createSnapshot(hdfs, dir1, "s0");
+
+ // move bar into dir1
+ final Path newBar = new Path(dir1, "newBar");
+ hdfs.rename(bar, newBar);
+ // move foo out of dir1 into dir2
+ final Path newFoo = new Path(dir2, "new");
+ hdfs.rename(foo, newFoo);
+
+ SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
+ verifyDiffReport(dir1, "s0", "s1",
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+ new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes(newBar
+ .getName())),
+ new DiffReportEntry(DiffType.DELETE,
+ DFSUtil.string2Bytes(foo.getName())));
+ }
+
+ /**
+ * Renaming a file/dir then delete the ancestor dir of the rename target
+ * should be reported as deleted.
+ */
+ @Test
+ public void testDiffReportWithRenameAndDelete() throws Exception {
+ final Path root = new Path("/");
+ final Path dir1 = new Path(root, "dir1");
+ final Path dir2 = new Path(root, "dir2");
+ final Path foo = new Path(dir1, "foo");
+ final Path fileInFoo = new Path(foo, "file");
+ final Path bar = new Path(dir2, "bar");
+ final Path fileInBar = new Path(bar, "file");
+ DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed);
+ DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, seed);
+
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
+ hdfs.rename(fileInFoo, fileInBar, Rename.OVERWRITE);
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+ verifyDiffReport(root, "s0", "s1",
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2/bar")),
+ new DiffReportEntry(DiffType.DELETE, DFSUtil
+ .string2Bytes("dir2/bar/file")),
+ new DiffReportEntry(DiffType.RENAME,
+ DFSUtil.string2Bytes("dir1/foo/file"),
+ DFSUtil.string2Bytes("dir2/bar/file")));
+
+ // delete bar
+ hdfs.delete(bar, true);
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s2");
+ verifyDiffReport(root, "s0", "s2",
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
+ new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("dir2/bar")),
+ new DiffReportEntry(DiffType.DELETE,
+ DFSUtil.string2Bytes("dir1/foo/file")));
+ }
+
+ @Test
+ public void testDiffReportWithRenameToNewDir() throws Exception {
+ final Path root = new Path("/");
+ final Path foo = new Path(root, "foo");
+ final Path fileInFoo = new Path(foo, "file");
+ DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed);
+
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
+ final Path bar = new Path(root, "bar");
+ hdfs.mkdirs(bar);
+ final Path fileInBar = new Path(bar, "file");
+ hdfs.rename(fileInFoo, fileInBar);
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+
+ verifyDiffReport(root, "s0", "s1",
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")),
+ new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("bar")),
+ new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo/file"),
+ DFSUtil.string2Bytes("bar/file")));
+ }
+
+ /**
+ * Rename a file and then append some data to it
+ */
+ @Test
+ public void testDiffReportWithRenameAndAppend() throws Exception {
+ final Path root = new Path("/");
+ final Path foo = new Path(root, "foo");
+ DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPLICATION, seed);
+
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
+ final Path bar = new Path(root, "bar");
+ hdfs.rename(foo, bar);
+ DFSTestUtil.appendFile(hdfs, bar, 10); // append 10 bytes
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+
+ // we always put modification on the file before rename
+ verifyDiffReport(root, "s0", "s1",
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
+ new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")),
+ new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"),
+ DFSUtil.string2Bytes("bar")));
+ }
}
\ No newline at end of file