You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2013/04/25 01:32:13 UTC
svn commit: r1471773 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src:
main/java/org/apache/hadoop/hdfs/server/namenode/
test/java/org/apache/hadoop/hdfs/server/namenode/
Author: atm
Date: Wed Apr 24 23:32:13 2013
New Revision: 1471773
URL: http://svn.apache.org/r1471773
Log:
HDFS-4739. NN can miscalculate the number of extra edit log segments to retain. Contributed by Aaron T. Myers.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java?rev=1471773&r1=1471772&r2=1471773&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java Wed Apr 24 23:32:13 2013
@@ -118,11 +118,17 @@ public class NNStorageRetentionManager {
.result();
}
});
+
+ // Remove from consideration any edit logs that are in fact required.
+ while (editLogs.size() > 0 &&
+ editLogs.get(editLogs.size() - 1).getFirstTxId() >= minimumRequiredTxId) {
+ editLogs.remove(editLogs.size() - 1);
+ }
// Next, adjust the number of transactions to retain if doing so would mean
// keeping too many segments around.
while (editLogs.size() > maxExtraEditsSegmentsToRetain) {
- purgeLogsFrom = editLogs.get(0).getFirstTxId();
+ purgeLogsFrom = editLogs.get(0).getLastTxId() + 1;
editLogs.remove(0);
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1471773&r1=1471772&r2=1471773&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java Wed Apr 24 23:32:13 2013
@@ -217,11 +217,12 @@ public class TestNNStorageRetentionManag
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(176, 200), true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 225), true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(226, 240), true);
- // Only retain 2 extra segments. The 301-400 segment is considered required,
- // not extra.
+ // Only retain 2 extra segments. The 301-350 and 351-400 segments are
+ // considered required, not extra.
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(241, 275), false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(276, 300), false);
- tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false);
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 350), false);
+ tc.addLog("/foo2/current/" + getFinalizedEditsFileName(351, 400), false);
tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
runTest(tc);
}