You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/07 21:02:03 UTC
[26/35] hadoop git commit: HDFS-13227. Add a method to calculate
cumulative diff over multiple snapshots in DirectoryDiffList. Contributed by
Shashikant Banerjee
HDFS-13227. Add a method to calculate cumulative diff over multiple snapshots in DirectoryDiffList. Contributed by Shashikant Banerjee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/346caa20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/346caa20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/346caa20
Branch: refs/heads/HDFS-7240
Commit: 346caa209571dedf1331b2658d5702b45dd40bfe
Parents: 871d0d3
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Tue Mar 6 13:35:03 2018 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Tue Mar 6 13:35:03 2018 -0800
----------------------------------------------------------------------
.../snapshot/AbstractINodeDiffList.java | 18 +++++-----
.../snapshot/DirectoryWithSnapshotFeature.java | 36 +++++++++++++++-----
2 files changed, 38 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/346caa20/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index 5163e5e..d115656 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -235,6 +235,12 @@ abstract class AbstractINodeDiffList<N extends INode,
return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId();
}
+ public final int getDiffIndexById(final int snapshotId) {
+ int diffIndex = diffs.binarySearch(snapshotId);
+ diffIndex = diffIndex < 0 ? (-diffIndex - 1) : diffIndex;
+ return diffIndex;
+ }
+
final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) {
if (diffs == null) {
return null;
@@ -247,10 +253,10 @@ abstract class AbstractINodeDiffList<N extends INode,
}
final int size = diffs.size();
- int earlierDiffIndex = diffs.binarySearch(earlier.getId());
+ int earlierDiffIndex = getDiffIndexById(earlier.getId());
int laterDiffIndex = later == null ? size
- : diffs.binarySearch(later.getId());
- if (-earlierDiffIndex - 1 == size) {
+ : getDiffIndexById(later.getId());
+ if (earlierDiffIndex == size) {
// if the earlierSnapshot is after the latest SnapshotDiff stored in
// diffs, no modification happened after the earlierSnapshot
return null;
@@ -260,10 +266,6 @@ abstract class AbstractINodeDiffList<N extends INode,
// before it, no modification happened before the laterSnapshot
return null;
}
- earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1)
- : earlierDiffIndex;
- laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1)
- : laterDiffIndex;
return new int[]{earlierDiffIndex, laterDiffIndex};
}
@@ -304,7 +306,7 @@ abstract class AbstractINodeDiffList<N extends INode,
}
return diff;
}
-
+
@Override
public Iterator<D> iterator() {
return diffs != null ? diffs.iterator() : Collections.emptyIterator();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/346caa20/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index b3ce602..19be8f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -228,12 +228,18 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
private List<INode> initChildren() {
if (children == null) {
final ChildrenDiff combined = new ChildrenDiff();
- for (DirectoryDiff d = DirectoryDiff.this; d != null;
- d = d.getPosterior()) {
+ DirectoryDiffList directoryDiffList =
+ currentDir.getDirectoryWithSnapshotFeature().diffs;
+ final int diffIndex =
+ directoryDiffList.getDiffIndexById(getSnapshotId());
+ List<DirectoryDiff> diffList = directoryDiffList
+ .getDiffListBetweenSnapshots(diffIndex,
+ directoryDiffList.asList().size(), currentDir);
+ for (DirectoryDiff d : diffList) {
combined.combinePosterior(d.diff, null);
}
- children = combined.apply2Current(ReadOnlyList.Util.asList(
- currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
+ children = combined.apply2Current(ReadOnlyList.Util
+ .asList(currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
}
return children;
}
@@ -383,6 +389,19 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
return NO_SNAPSHOT_ID;
}
+
+ /**
+ * Returns the list of diffs between two indexes corresponding to two
+ * snapshots.
+ * @param fromIndex Index of the diff corresponding to the earlier snapshot
+ * @param toIndex Index of the diff corresponding to the later snapshot
+ * @param dir The Directory to which the diffList belongs
+ * @return list of directory diffs
+ */
+ List<DirectoryDiff> getDiffListBetweenSnapshots(int fromIndex, int toIndex,
+ INodeDirectory dir) {
+ return asList().getMinListForRange(fromIndex, toIndex, dir);
+ }
}
private static Map<INode, INode> cloneDiffList(List<INode> diffList) {
@@ -597,7 +616,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
return diff != null ? diff.getChild(name, true, currentINode)
: currentINode.getChild(name, Snapshot.CURRENT_STATE_ID);
}
-
+
/** Used to record the modification of a symlink node */
public INode saveChild2Snapshot(INodeDirectory currentINode,
final INode child, final int latestSnapshotId, final INode snapshotCopy) {
@@ -678,9 +697,10 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
boolean dirMetadataChanged = false;
INodeDirectoryAttributes dirCopy = null;
- DiffList<DirectoryDiff> difflist = diffs.asList();
- for (int i = earlierDiffIndex; i < laterDiffIndex; i++) {
- DirectoryDiff sdiff = difflist.get(i);
+ List<DirectoryDiff> difflist = diffs
+ .getDiffListBetweenSnapshots(earlierDiffIndex, laterDiffIndex,
+ currentINode);
+ for (DirectoryDiff sdiff : difflist) {
diff.combinePosterior(sdiff.diff, null);
if (!dirMetadataChanged && sdiff.snapshotINode != null) {
if (dirCopy == null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org