You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by so...@apache.org on 2021/10/18 11:50:30 UTC
[hadoop] branch branch-3.2.3 updated: HDFS-16272. Fix int overflow
in computing safe length during EC block recovery (#3548)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2.3 by this push:
new e1f11d5 HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)
e1f11d5 is described below
commit e1f11d5f63ac69328055b971e6b80fc9797561a8
Author: daimin <da...@outlook.com>
AuthorDate: Mon Oct 18 18:55:55 2021 +0800
HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)
(cherry picked from commit 5337bebcc53a0144311877362787ed9ece0cb118)
---
.../org/apache/hadoop/hdfs/util/StripedBlockUtil.java | 7 +++----
.../apache/hadoop/hdfs/TestLeaseRecoveryStriped.java | 18 ++++++++++++++++++
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 718c51d..4122636 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -245,8 +245,7 @@ public class StripedBlockUtil {
Arrays.sort(cpy);
// full stripe is a stripe has at least dataBlkNum full cells.
// lastFullStripeIdx is the index of the last full stripe.
- int lastFullStripeIdx =
- (int) (cpy[cpy.length - dataBlkNum] / cellSize);
+ long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
return lastFullStripeIdx * stripeSize; // return the safeLength
// TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
// such a stripe (and it must be partial).
@@ -271,9 +270,9 @@ public class StripedBlockUtil {
*/
public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
long offsetInBlk, int idxInBlockGroup) {
- int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+ long cellIdxInBlk = offsetInBlk / cellSize;
return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
- + idxInBlockGroup * cellSize // m full cells before offset
+ + (long)idxInBlockGroup * cellSize // m full cells before offset
+ offsetInBlk % cellSize; // partial cell
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 36a3055..2193932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -188,6 +188,24 @@ public class TestLeaseRecoveryStriped {
}
}
+ @Test
+ public void testSafeLength() {
+ checkSafeLength(0, 0); // Length of: 0
+ checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
+ checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
+ checkSafeLength(189729792, 1132462080L); // Length of: 189729792
+ checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
+ checkSafeLength(517399040, 3101687808L); // Length of: 517399040
+ checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
+ }
+
+ private void checkSafeLength(int blockLength, long expectedSafeLength) {
+ int[] blockLengths = new int[]{blockLength, blockLength, blockLength, blockLength,
+ blockLength, blockLength};
+ long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
+ Assert.assertEquals(expectedSafeLength, safeLength);
+ }
+
private void runTest(int[] blockLengths, long safeLength) throws Exception {
writePartialBlocks(blockLengths);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org