You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2018/11/02 09:29:52 UTC
[44/50] [abbrv] hadoop git commit: HDFS-14008. NN should log
snapshotdiff report. Contributed by Pranay Singh.
HDFS-14008. NN should log snapshotdiff report. Contributed by Pranay Singh.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d98b881e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d98b881e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d98b881e
Branch: refs/heads/HDFS-13891
Commit: d98b881e9ab826eb7b70485d0de2a41ab7345334
Parents: 2ab611d
Author: Pranay Singh <ps...@cloudera.com>
Authored: Thu Nov 1 17:25:11 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Nov 1 17:26:00 2018 -0700
----------------------------------------------------------------------
.../hdfs/protocol/SnapshotDiffReport.java | 65 ++++++++++++++++++++
.../hdfs/server/namenode/FSNamesystem.java | 20 ++++++
.../snapshot/DirectorySnapshottableFeature.java | 5 ++
.../namenode/snapshot/SnapshotDiffInfo.java | 50 ++++++++++++++-
4 files changed, 139 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
index 8ee4ec7..7bc95c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
@@ -170,14 +170,75 @@ public class SnapshotDiffReport {
/** end point of the diff */
private final String toSnapshot;
+
/** list of diff */
private final List<DiffReportEntry> diffList;
+ /**
+ * Records the stats related to Snapshot diff operation.
+ */
+ public static class DiffStats {
+ // Total dirs processed
+ private long totalDirsProcessed;
+
+ // Total dirs compared
+ private long totalDirsCompared;
+
+ // Total files processed
+ private long totalFilesProcessed;
+
+ // Total files compared
+ private long totalFilesCompared;
+
+ // Total children listing time
+ private final long totalChildrenListingTime;
+
+ public DiffStats(long totalDirsProcessed, long totalDirsCompared,
+ long totalFilesProcessed, long totalFilesCompared,
+ long totalChildrenListingTime) {
+ this.totalDirsCompared = totalDirsProcessed;
+ this.totalDirsProcessed = totalDirsCompared;
+ this.totalFilesCompared = totalFilesProcessed;
+ this.totalFilesProcessed = totalFilesCompared;
+ this.totalChildrenListingTime = totalChildrenListingTime;
+ }
+
+ public long getTotalDirsProcessed() {
+ return this.totalDirsProcessed;
+ }
+
+ public long getTotalDirsCompared() {
+ return this.totalDirsCompared;
+ }
+
+ public long getTotalFilesProcessed() {
+ return this.totalFilesProcessed;
+ }
+
+ public long getTotalFilesCompared() {
+ return this.totalFilesCompared;
+ }
+
+ public long getTotalChildrenListingTime() {
+ return totalChildrenListingTime;
+ }
+ }
+
+ /* Stats associated with the SnapshotDiff Report. */
+ private final DiffStats diffStats;
+
public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
String toSnapshot, List<DiffReportEntry> entryList) {
+ this(snapshotRoot, fromSnapshot, toSnapshot, new DiffStats(0, 0, 0, 0, 0),
+ entryList);
+ }
+
+ public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
+ String toSnapshot, DiffStats dStat, List<DiffReportEntry> entryList) {
this.snapshotRoot = snapshotRoot;
this.fromSnapshot = fromSnapshot;
this.toSnapshot = toSnapshot;
+ this.diffStats = dStat;
this.diffList = entryList != null ? entryList : Collections
.<DiffReportEntry> emptyList();
}
@@ -197,6 +258,10 @@ public class SnapshotDiffReport {
return toSnapshot;
}
+ public DiffStats getStats() {
+ return this.diffStats;
+ }
+
/** @return {@link #diffList} */
public List<DiffReportEntry> getDiffList() {
return diffList;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d1904fa..8b1cdf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean;
import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
+import org.apache.hadoop.util.Time;
import static org.apache.hadoop.util.Time.now;
import static org.apache.hadoop.util.Time.monotonicNow;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
@@ -6624,6 +6625,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException {
+ long begTime = Time.monotonicNow();
final String operationName = "computeSnapshotDiff";
SnapshotDiffReport diffs = null;
checkOperation(OperationCategory.READ);
@@ -6634,6 +6636,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
path : Snapshot.getSnapshotPath(path, toSnapshot);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
+ long actualTime = Time.monotonicNow();
try {
checkOperation(OperationCategory.READ);
diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, pc, snapshotManager,
@@ -6646,6 +6649,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} finally {
readUnlock(operationName);
}
+
+ if (diffs != null) {
+ SnapshotDiffReport.DiffStats dstat = diffs.getStats();
+ LOG.info("SnapshotDiffReport '"
+ + fromSnapshot
+ + "' to '" + toSnapshot + "'. Total comparison dirs: "
+ + dstat.getTotalDirsCompared()
+ + "/" + dstat.getTotalDirsProcessed()
+ + ", files: "
+ + dstat.getTotalFilesCompared()
+ + "/" + dstat.getTotalFilesProcessed()
+ + ". Time snapChildrenListing: "
+ + dstat.getTotalChildrenListingTime() / 1000.0 + "s, actual: "
+ + ((Time.monotonicNow() - actualTime) / 1000.0) + "s, total: "
+ + ((Time.monotonicNow() - begTime) / 1000.0) + "s.");
+ }
+
logAuditEvent(success, operationName, fromSnapshotRoot,
toSnapshotRoot, null);
return diffs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index 15aa22a..073b88b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -389,9 +389,13 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
if (change) {
diffReport.addDirDiff(dir, relativePath, diff);
}
+ } else {
+ diffReport.incrementDirsProcessed();
}
+ long startTime = Time.monotonicNow();
ReadOnlyList<INode> children = dir.getChildrenList(earlierSnapshot
.getId());
+ diffReport.addChildrenListingTime(Time.monotonicNow() - startTime);
for (INode child : children) {
final byte[] name = child.getLocalNameBytes();
boolean toProcess = !diff.containsDeleted(name);
@@ -418,6 +422,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
if (change) {
diffReport.addFileDiff(file, relativePath);
}
+ diffReport.incrementFilesProcessed();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
index 4c74afd..ab6f415 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
@@ -125,6 +125,21 @@ class SnapshotDiffInfo {
private final Map<Long, RenameEntry> renameMap =
new HashMap<Long, RenameEntry>();
+ // Total directories compared
+ private long totalDirsCompared;
+
+ // Total directories
+ private long totalDirsProcessed;
+
+ // Total files compared
+ private long totalFilesCompared;
+
+ // Total files
+ private long totalFilesProcessed;
+
+ // Total children listing time
+ private long childrenListingTime;
+
SnapshotDiffInfo(INodeDirectory snapshotRootDir,
INodeDirectory snapshotDiffScopeDir, Snapshot start, Snapshot end) {
Preconditions.checkArgument(snapshotRootDir.isSnapshottable() &&
@@ -133,6 +148,10 @@ class SnapshotDiffInfo {
this.snapshotDiffScopeDir = snapshotDiffScopeDir;
this.from = start;
this.to = end;
+ this.totalDirsCompared = 0;
+ this.totalDirsProcessed = 0;
+ this.totalFilesCompared = 0;
+ this.totalFilesProcessed = 0;
}
/** Add a dir-diff pair */
@@ -164,6 +183,29 @@ class SnapshotDiffInfo {
return to;
}
+
+ void incrementDirsCompared() {
+ this.totalDirsCompared++;
+ incrementDirsProcessed();
+ }
+
+ void incrementDirsProcessed() {
+ this.totalDirsProcessed++;
+ }
+
+ void incrementFilesCompared() {
+ this.totalFilesCompared++;
+ incrementFilesProcessed();
+ }
+
+ void incrementFilesProcessed() {
+ this.totalFilesProcessed++;
+ }
+
+ public void addChildrenListingTime(long millis) {
+ this.childrenListingTime += millis;
+ }
+
private RenameEntry getEntry(long inodeId) {
RenameEntry entry = renameMap.get(inodeId);
if (entry == null) {
@@ -203,9 +245,15 @@ class SnapshotDiffInfo {
diffReportList.addAll(subList);
}
}
+
+ SnapshotDiffReport.DiffStats dStats = new SnapshotDiffReport.DiffStats(
+ this.totalDirsCompared, this.totalDirsProcessed,
+ this.totalFilesCompared, this.totalFilesProcessed,
+ this.childrenListingTime);
+
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
- diffReportList);
+ dStats, diffReportList);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org