You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2020/08/27 09:25:23 UTC
[hadoop] branch trunk updated: HDFS-15500. In-order deletion of
snapshots: Diff lists must be update only in the last snapshot. (#2233)
This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 41182a9 HDFS-15500. In-order deletion of snapshots: Diff lists must be update only in the last snapshot. (#2233)
41182a9 is described below
commit 41182a9b6d81d0c8a4dc0a9cf89ea0ade815afd3
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Thu Aug 27 02:24:52 2020 -0700
HDFS-15500. In-order deletion of snapshots: Diff lists must be update only in the last snapshot. (#2233)
---
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 6 ++++++
.../hdfs/server/namenode/snapshot/AbstractINodeDiffList.java | 6 +++++-
.../hdfs/server/namenode/snapshot/DiffListByArrayList.java | 2 ++
.../namenode/snapshot/DirectoryWithSnapshotFeature.java | 2 ++
.../hdfs/server/namenode/snapshot/SnapshotManager.java | 12 ++++++++++++
5 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 15bf6b1..badf237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1569,6 +1569,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// null in some unit tests
haContext.checkOperation(op);
}
+
+ boolean assertsEnabled = false;
+ assert assertsEnabled = true; // Intentional side effect!!!
+ if (assertsEnabled && op == OperationCategory.WRITE) {
+ getSnapshotManager().initThreadLocals();
+ }
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index 776adf1..16e3b75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -76,7 +76,11 @@ abstract class AbstractINodeDiffList<N extends INode,
if (diffs == null) {
return;
}
- int snapshotIndex = diffs.binarySearch(snapshot);
+ final int snapshotIndex = diffs.binarySearch(snapshot);
+ // DeletionOrdered: only can remove the element at index 0 and no prior
+ // check snapshotIndex <= 0 since the diff may not exist
+ assert !SnapshotManager.isDeletionOrdered()
+ || (snapshotIndex <= 0 && prior == Snapshot.NO_SNAPSHOT_ID);
D removed;
if (snapshotIndex == 0) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
index 95c23df..7fa3f05 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java
@@ -57,6 +57,8 @@ public class DiffListByArrayList<T extends Comparable<Integer>>
@Override
public T remove(int i) {
+ // DeletionOrdered: only can remove the element at index 0
+ assert !SnapshotManager.isDeletionOrdered() || i == 0;
return list.remove(i);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index b8f7b65..c3a9aa1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -175,6 +175,8 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
final INode.ReclaimContext reclaimContext,
final INodeDirectory currentDir,
final DirectoryDiff posterior) {
+ // DeletionOrdered: must not combine posterior
+ assert !SnapshotManager.isDeletionOrdered();
diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
/** Collect blocks for deleted files. */
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
index b5b0971..789fa3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
@@ -95,6 +95,18 @@ public class SnapshotManager implements SnapshotStatsMXBean {
static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT
= 5 * 60_000L; //5 minutes
+ private static final ThreadLocal<Boolean> DELETION_ORDERED
+ = new ThreadLocal<>();
+
+ static boolean isDeletionOrdered() {
+ final Boolean b = DELETION_ORDERED.get();
+ return b != null? b: false;
+ }
+
+ public void initThreadLocals() {
+ DELETION_ORDERED.set(isSnapshotDeletionOrdered());
+ }
+
private final FSDirectory fsdir;
private boolean captureOpenFiles;
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org