You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ma...@apache.org on 2011/06/10 02:55:32 UTC
svn commit: r1134157 - in /hadoop/hdfs/branches/yahoo-merge: ./ CHANGES.txt
src/java/ src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
Author: mattf
Date: Fri Jun 10 00:55:32 2011
New Revision: 1134157
URL: http://svn.apache.org/viewvc?rev=1134157&view=rev
Log:
HDFS-900. Merge to yahoo-merge.
Modified:
hadoop/hdfs/branches/yahoo-merge/ (props changed)
hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
hadoop/hdfs/branches/yahoo-merge/src/java/ (props changed)
hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jun 10 00:55:32 2011
@@ -3,4 +3,4 @@
/hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067079,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138
Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1134157&r1=1134156&r2=1134157&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Fri Jun 10 00:55:32 2011
@@ -887,6 +887,8 @@ Release 0.22.0 - Unreleased
HDFS-1503. TestSaveNamespace fails. (Todd Lipcon via cos)
+ HDFS-900. Corrupt replicas are not processed correctly in block report (shv)
+
Release 0.21.1 - Unreleased
HDFS-1411. Correct backup node startup command in hdfs user guide.
Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jun 10 00:55:32 2011
@@ -4,4 +4,4 @@
/hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1079607,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1038001,1038859,1039957,1040005,1040411-1040412,1042017,1042925,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1066305,1067079,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085460,1085509,1086479,1086654,1086693,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092534,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1099972,1100279,1101137,1101282,1101293,1101324,1101343,1101675,1101753,1102005,1102459,1102511,1102833,1102938,1102947,1103957-1103958,1103970,1104395,1104407,1104568,1124349,1124576,1125028,1125145,1125189,1127712,1127759,1128534,1129942,1130262,1130339,1130367-1130368,1130381,1130667,1130855,1130870,1131124,1131264,1131331,1132698,1132829,1133181,1134138
Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=1134157&r1=1134156&r2=1134157&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Fri Jun 10 00:55:32 2011
@@ -399,8 +399,7 @@ public class BlockManager {
blocksMap.nodeIterator(block); it.hasNext();) {
String storageID = it.next().getStorageID();
// filter invalidate replicas
- Collection<Block> blocks = recentInvalidateSets.get(storageID);
- if(blocks==null || !blocks.contains(block)) {
+ if( ! belongsToInvalidates(storageID, block)) {
machineSet.add(storageID);
}
}
@@ -498,16 +497,21 @@ public class BlockManager {
minReplication);
}
- void removeFromInvalidates(String datanodeId, Block block) {
- Collection<Block> v = recentInvalidateSets.get(datanodeId);
+ void removeFromInvalidates(String storageID, Block block) {
+ Collection<Block> v = recentInvalidateSets.get(storageID);
if (v != null && v.remove(block)) {
pendingDeletionBlocksCount--;
if (v.isEmpty()) {
- recentInvalidateSets.remove(datanodeId);
+ recentInvalidateSets.remove(storageID);
}
}
}
+ boolean belongsToInvalidates(String storageID, Block block) {
+ Collection<Block> invalidateSet = recentInvalidateSets.get(storageID);
+ return invalidateSet != null && invalidateSet.contains(block);
+ }
+
/**
* Adds block to list of blocks which will be invalidated on specified
* datanode
Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java?rev=1134157&r1=1134156&r2=1134157&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java (original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java Fri Jun 10 00:55:32 2011
@@ -498,6 +498,13 @@ public class DatanodeDescriptor extends
storedBlock.getBlockUCState());
}
+ // Ignore replicas already scheduled to be removed from the DN
+ if(blockManager.belongsToInvalidates(getStorageID(), block)) {
+ assert storedBlock.findDatanode(this) < 0 : "Block " + block
+ + " in recentInvalidatesSet should not appear in DN " + this;
+ return storedBlock;
+ }
+
// Block is on the DN
boolean isCorrupt = false;
switch(rState) {