You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wh...@apache.org on 2014/06/24 21:59:35 UTC
svn commit: r1605169 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/
Author: wheat9
Date: Tue Jun 24 19:59:34 2014
New Revision: 1605169
URL: http://svn.apache.org/r1605169
Log:
HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable. Contributed by Jing Zhao.
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1605169&r1=1605168&r2=1605169&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jun 24 19:59:34 2014
@@ -473,6 +473,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu)
+ HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable.
+ (Jing Zhao via wheat9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java?rev=1605169&r1=1605168&r2=1605169&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java Tue Jun 24 19:59:34 2014
@@ -23,16 +23,14 @@ import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import com.google.common.base.Objects;
/**
* This class represents to end users the difference between two snapshots of
* the same directory, or the difference between a snapshot of the directory and
- * its current state. Instead of capturing all the details of the diff, which
- * is stored in {@link SnapshotDiffInfo}, this class only lists where the
- * changes happened and their types.
+ * its current state. Instead of capturing all the details of the diff, this
+ * class only lists where the changes happened and their types.
*/
public class SnapshotDiffReport {
private final static String LINE_SEPARATOR = System.getProperty(
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1605169&r1=1605168&r2=1605169&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Jun 24 19:59:34 2014
@@ -85,7 +85,18 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.util.Time.now;
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
@@ -170,7 +181,6 @@ import org.apache.hadoop.hdfs.protocol.R
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
@@ -208,7 +218,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -7484,7 +7493,7 @@ public class FSNamesystem implements Nam
*/
SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException {
- SnapshotDiffInfo diffs = null;
+ SnapshotDiffReport diffs;
checkOperation(OperationCategory.READ);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
@@ -7498,13 +7507,11 @@ public class FSNamesystem implements Nam
} finally {
readUnlock();
}
-
+
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
}
- return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
- path, fromSnapshot, toSnapshot,
- Collections.<DiffReportEntry> emptyList());
+ return diffs;
}
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1605169&r1=1605168&r2=1605169&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java Tue Jun 24 19:59:34 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.na
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayDeque;
-import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
@@ -29,8 +28,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
@@ -41,7 +38,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.Quota;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.Diff;
import org.apache.hadoop.hdfs.util.Diff.Container;
@@ -161,43 +157,6 @@ public class DirectoryWithSnapshotFeatur
}
}
}
-
- /**
- * Interpret the diff and generate a list of {@link DiffReportEntry}.
- * @param parentPath The relative path of the parent.
- * @param fromEarlier True indicates {@code diff=later-earlier},
- * False indicates {@code diff=earlier-later}
- * @return A list of {@link DiffReportEntry} as the diff report.
- */
- public List<DiffReportEntry> generateReport(byte[][] parentPath,
- boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
- List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
- List<INode> created = getList(ListType.CREATED);
- List<INode> deleted = getList(ListType.DELETED);
- byte[][] fullPath = new byte[parentPath.length + 1][];
- System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
- for (INode cnode : created) {
- RenameEntry entry = renameMap.get(cnode.getId());
- if (entry == null || !entry.isRename()) {
- fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
- list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
- : DiffType.DELETE, fullPath));
- }
- }
- for (INode dnode : deleted) {
- RenameEntry entry = renameMap.get(dnode.getId());
- if (entry != null && entry.isRename()) {
- list.add(new DiffReportEntry(DiffType.RENAME,
- fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
- fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
- } else {
- fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
- list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
- : DiffType.CREATE, fullPath));
- }
- }
- return list;
- }
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1605169&r1=1605168&r2=1605169&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java Tue Jun 24 19:59:34 2014
@@ -21,22 +21,14 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
@@ -56,7 +48,6 @@ import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.google.common.primitives.SignedBytes;
/**
* Directories where taking snapshots is allowed.
@@ -79,164 +70,6 @@ public class INodeDirectorySnapshottable
}
return (INodeDirectorySnapshottable)dir;
}
-
- /**
- * A class describing the difference between snapshots of a snapshottable
- * directory.
- */
- public static class SnapshotDiffInfo {
- /** Compare two inodes based on their full names */
- public static final Comparator<INode> INODE_COMPARATOR =
- new Comparator<INode>() {
- @Override
- public int compare(INode left, INode right) {
- if (left == null) {
- return right == null ? 0 : -1;
- } else {
- if (right == null) {
- return 1;
- } else {
- int cmp = compare(left.getParent(), right.getParent());
- return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
- left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
- }
- }
- }
- };
-
- static class RenameEntry {
- private byte[][] sourcePath;
- private byte[][] targetPath;
-
- void setSource(INode source, byte[][] sourceParentPath) {
- Preconditions.checkState(sourcePath == null);
- sourcePath = new byte[sourceParentPath.length + 1][];
- System.arraycopy(sourceParentPath, 0, sourcePath, 0,
- sourceParentPath.length);
- sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
- }
-
- void setTarget(INode target, byte[][] targetParentPath) {
- targetPath = new byte[targetParentPath.length + 1][];
- System.arraycopy(targetParentPath, 0, targetPath, 0,
- targetParentPath.length);
- targetPath[targetPath.length - 1] = target.getLocalNameBytes();
- }
-
- void setTarget(byte[][] targetPath) {
- this.targetPath = targetPath;
- }
-
- boolean isRename() {
- return sourcePath != null && targetPath != null;
- }
-
- byte[][] getSourcePath() {
- return sourcePath;
- }
-
- byte[][] getTargetPath() {
- return targetPath;
- }
- }
-
- /** The root directory of the snapshots */
- private final INodeDirectorySnapshottable snapshotRoot;
- /** The starting point of the difference */
- private final Snapshot from;
- /** The end point of the difference */
- private final Snapshot to;
- /**
- * A map recording modified INodeFile and INodeDirectory and their relative
- * path corresponding to the snapshot root. Sorted based on their names.
- */
- private final SortedMap<INode, byte[][]> diffMap =
- new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
- /**
- * A map capturing the detailed difference about file creation/deletion.
- * Each key indicates a directory whose children have been changed between
- * the two snapshots, while its associated value is a {@link ChildrenDiff}
- * storing the changes (creation/deletion) happened to the children (files).
- */
- private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
- new HashMap<INodeDirectory, ChildrenDiff>();
-
- private final Map<Long, RenameEntry> renameMap =
- new HashMap<Long, RenameEntry>();
-
- SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
- Snapshot end) {
- this.snapshotRoot = snapshotRoot;
- this.from = start;
- this.to = end;
- }
-
- /** Add a dir-diff pair */
- private void addDirDiff(INodeDirectory dir, byte[][] relativePath,
- ChildrenDiff diff) {
- dirDiffMap.put(dir, diff);
- diffMap.put(dir, relativePath);
- // detect rename
- for (INode created : diff.getList(ListType.CREATED)) {
- if (created.isReference()) {
- RenameEntry entry = getEntry(created.getId());
- if (entry.getTargetPath() == null) {
- entry.setTarget(created, relativePath);
- }
- }
- }
- for (INode deleted : diff.getList(ListType.DELETED)) {
- if (deleted instanceof INodeReference.WithName) {
- RenameEntry entry = getEntry(deleted.getId());
- entry.setSource(deleted, relativePath);
- }
- }
- }
-
- private RenameEntry getEntry(long inodeId) {
- RenameEntry entry = renameMap.get(inodeId);
- if (entry == null) {
- entry = new RenameEntry();
- renameMap.put(inodeId, entry);
- }
- return entry;
- }
-
- private void setRenameTarget(long inodeId, byte[][] path) {
- getEntry(inodeId).setTarget(path);
- }
-
- /** Add a modified file */
- private void addFileDiff(INodeFile file, byte[][] relativePath) {
- diffMap.put(file, relativePath);
- }
-
- /** @return True if {@link #from} is earlier than {@link #to} */
- private boolean isFromEarlier() {
- return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
- }
-
- /**
- * Generate a {@link SnapshotDiffReport} based on detailed diff information.
- * @return A {@link SnapshotDiffReport} describing the difference
- */
- public SnapshotDiffReport generateReport() {
- List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
- for (INode node : diffMap.keySet()) {
- diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
- .get(node), null));
- if (node.isDirectory()) {
- ChildrenDiff dirDiff = dirDiffMap.get(node);
- List<DiffReportEntry> subList = dirDiff.generateReport(
- diffMap.get(node), isFromEarlier(), renameMap);
- diffReportList.addAll(subList);
- }
- }
- return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
- Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
- diffReportList);
- }
- }
/**
* Snapshots of this directory in ascending order of snapshot names.
@@ -496,9 +329,9 @@ public class INodeDirectorySnapshottable
private void computeDiffRecursively(INode node, List<byte[]> parentPath,
SnapshotDiffInfo diffReport) {
final Snapshot earlierSnapshot = diffReport.isFromEarlier() ?
- diffReport.from : diffReport.to;
+ diffReport.getFrom() : diffReport.getTo();
final Snapshot laterSnapshot = diffReport.isFromEarlier() ?
- diffReport.to : diffReport.from;
+ diffReport.getTo() : diffReport.getFrom();
byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
if (node.isDirectory()) {
final ChildrenDiff diff = new ChildrenDiff();
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java?rev=1605169&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java Tue Jun 24 19:59:34 2014
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.INodeReference;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
+import org.apache.hadoop.hdfs.util.Diff.ListType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.SignedBytes;
+
+/**
+ * A class describing the difference between snapshots of a snapshottable
+ * directory.
+ */
+class SnapshotDiffInfo {
+ /** Compare two inodes based on their full names */
+ public static final Comparator<INode> INODE_COMPARATOR =
+ new Comparator<INode>() {
+ @Override
+ public int compare(INode left, INode right) {
+ if (left == null) {
+ return right == null ? 0 : -1;
+ } else {
+ if (right == null) {
+ return 1;
+ } else {
+ int cmp = compare(left.getParent(), right.getParent());
+ return cmp == 0 ? SignedBytes.lexicographicalComparator().compare(
+ left.getLocalNameBytes(), right.getLocalNameBytes()) : cmp;
+ }
+ }
+ }
+ };
+
+ static class RenameEntry {
+ private byte[][] sourcePath;
+ private byte[][] targetPath;
+
+ void setSource(INode source, byte[][] sourceParentPath) {
+ Preconditions.checkState(sourcePath == null);
+ sourcePath = new byte[sourceParentPath.length + 1][];
+ System.arraycopy(sourceParentPath, 0, sourcePath, 0,
+ sourceParentPath.length);
+ sourcePath[sourcePath.length - 1] = source.getLocalNameBytes();
+ }
+
+ void setTarget(INode target, byte[][] targetParentPath) {
+ targetPath = new byte[targetParentPath.length + 1][];
+ System.arraycopy(targetParentPath, 0, targetPath, 0,
+ targetParentPath.length);
+ targetPath[targetPath.length - 1] = target.getLocalNameBytes();
+ }
+
+ void setTarget(byte[][] targetPath) {
+ this.targetPath = targetPath;
+ }
+
+ boolean isRename() {
+ return sourcePath != null && targetPath != null;
+ }
+
+ byte[][] getSourcePath() {
+ return sourcePath;
+ }
+
+ byte[][] getTargetPath() {
+ return targetPath;
+ }
+ }
+
+ /** The root directory of the snapshots */
+ private final INodeDirectorySnapshottable snapshotRoot;
+ /** The starting point of the difference */
+ private final Snapshot from;
+ /** The end point of the difference */
+ private final Snapshot to;
+ /**
+ * A map recording modified INodeFile and INodeDirectory and their relative
+ * path corresponding to the snapshot root. Sorted based on their names.
+ */
+ private final SortedMap<INode, byte[][]> diffMap =
+ new TreeMap<INode, byte[][]>(INODE_COMPARATOR);
+ /**
+ * A map capturing the detailed difference about file creation/deletion.
+ * Each key indicates a directory whose children have been changed between
+ * the two snapshots, while its associated value is a {@link ChildrenDiff}
+ * storing the changes (creation/deletion) happened to the children (files).
+ */
+ private final Map<INodeDirectory, ChildrenDiff> dirDiffMap =
+ new HashMap<INodeDirectory, ChildrenDiff>();
+
+ private final Map<Long, RenameEntry> renameMap =
+ new HashMap<Long, RenameEntry>();
+
+ SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start,
+ Snapshot end) {
+ this.snapshotRoot = snapshotRoot;
+ this.from = start;
+ this.to = end;
+ }
+
+ /** Add a dir-diff pair */
+ void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) {
+ dirDiffMap.put(dir, diff);
+ diffMap.put(dir, relativePath);
+ // detect rename
+ for (INode created : diff.getList(ListType.CREATED)) {
+ if (created.isReference()) {
+ RenameEntry entry = getEntry(created.getId());
+ if (entry.getTargetPath() == null) {
+ entry.setTarget(created, relativePath);
+ }
+ }
+ }
+ for (INode deleted : diff.getList(ListType.DELETED)) {
+ if (deleted instanceof INodeReference.WithName) {
+ RenameEntry entry = getEntry(deleted.getId());
+ entry.setSource(deleted, relativePath);
+ }
+ }
+ }
+
+ Snapshot getFrom() {
+ return from;
+ }
+
+ Snapshot getTo() {
+ return to;
+ }
+
+ private RenameEntry getEntry(long inodeId) {
+ RenameEntry entry = renameMap.get(inodeId);
+ if (entry == null) {
+ entry = new RenameEntry();
+ renameMap.put(inodeId, entry);
+ }
+ return entry;
+ }
+
+ void setRenameTarget(long inodeId, byte[][] path) {
+ getEntry(inodeId).setTarget(path);
+ }
+
+ /** Add a modified file */
+ void addFileDiff(INodeFile file, byte[][] relativePath) {
+ diffMap.put(file, relativePath);
+ }
+
+ /** @return True if {@link #from} is earlier than {@link #to} */
+ boolean isFromEarlier() {
+ return Snapshot.ID_COMPARATOR.compare(from, to) < 0;
+ }
+
+ /**
+ * Generate a {@link SnapshotDiffReport} based on detailed diff information.
+ * @return A {@link SnapshotDiffReport} describing the difference
+ */
+ public SnapshotDiffReport generateReport() {
+ List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
+ for (INode node : diffMap.keySet()) {
+ diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
+ .get(node), null));
+ if (node.isDirectory()) {
+ List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node),
+ diffMap.get(node), isFromEarlier(), renameMap);
+ diffReportList.addAll(subList);
+ }
+ }
+ return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
+ Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
+ diffReportList);
+ }
+
+ /**
+ * Interpret the ChildrenDiff and generate a list of {@link DiffReportEntry}.
+ * @param dirDiff The ChildrenDiff.
+ * @param parentPath The relative path of the parent.
+ * @param fromEarlier True indicates {@code diff=later-earlier},
+ * False indicates {@code diff=earlier-later}
+ * @param renameMap A map containing information about rename operations.
+ * @return A list of {@link DiffReportEntry} as the diff report.
+ */
+ private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff,
+ byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) {
+ List<DiffReportEntry> list = new ArrayList<DiffReportEntry>();
+ List<INode> created = dirDiff.getList(ListType.CREATED);
+ List<INode> deleted = dirDiff.getList(ListType.DELETED);
+ byte[][] fullPath = new byte[parentPath.length + 1][];
+ System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length);
+ for (INode cnode : created) {
+ RenameEntry entry = renameMap.get(cnode.getId());
+ if (entry == null || !entry.isRename()) {
+ fullPath[fullPath.length - 1] = cnode.getLocalNameBytes();
+ list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE
+ : DiffType.DELETE, fullPath));
+ }
+ }
+ for (INode dnode : deleted) {
+ RenameEntry entry = renameMap.get(dnode.getId());
+ if (entry != null && entry.isRename()) {
+ list.add(new DiffReportEntry(DiffType.RENAME,
+ fromEarlier ? entry.getSourcePath() : entry.getTargetPath(),
+ fromEarlier ? entry.getTargetPath() : entry.getSourcePath()));
+ } else {
+ fullPath[fullPath.length - 1] = dnode.getLocalNameBytes();
+ list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE
+ : DiffType.CREATE, fullPath));
+ }
+ }
+ return list;
+ }
+}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1605169&r1=1605168&r2=1605169&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Tue Jun 24 19:59:34 2014
@@ -30,9 +30,11 @@ import java.util.concurrent.atomic.Atomi
import javax.management.ObjectName;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -40,7 +42,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import org.apache.hadoop.metrics2.util.MBeans;
/**
@@ -361,12 +362,13 @@ public class SnapshotManager implements
* Compute the difference between two snapshots of a directory, or between a
* snapshot of the directory and its current tree.
*/
- public SnapshotDiffInfo diff(final String path, final String from,
+ public SnapshotDiffReport diff(final String path, final String from,
final String to) throws IOException {
if ((from == null || from.isEmpty())
&& (to == null || to.isEmpty())) {
// both fromSnapshot and toSnapshot indicate the current tree
- return null;
+ return new SnapshotDiffReport(path, from, to,
+ Collections.<DiffReportEntry> emptyList());
}
// Find the source root directory path where the snapshots were taken.
@@ -374,8 +376,10 @@ public class SnapshotManager implements
INodesInPath inodesInPath = fsdir.getINodesInPath4Write(path.toString());
final INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
.valueOf(inodesInPath.getLastINode(), path);
-
- return snapshotRoot.computeDiff(from, to);
+
+ final SnapshotDiffInfo diffs = snapshotRoot.computeDiff(from, to);
+ return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
+ path, from, to, Collections.<DiffReportEntry> emptyList());
}
public void clearSnapshottableDirs() {