You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ck...@apache.org on 2022/10/10 08:08:36 UTC
[ozone] branch master updated: HDDS-7302. Fix inaccurate container block count caused by missing block (#3814)
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 69d4dbd95a HDDS-7302. Fix inaccurate container block count caused by missing block (#3814)
69d4dbd95a is described below
commit 69d4dbd95a242efe0ccd8d5c4113dd0c4825fcab
Author: Nibiru <ax...@qq.com>
AuthorDate: Mon Oct 10 16:08:27 2022 +0800
HDDS-7302. Fix inaccurate container block count caused by missing block (#3814)
---
.../background/BlockDeletingService.java | 47 ++++++++++++++++++----
.../container/common/TestBlockDeletingService.java | 6 +++
2 files changed, 45 insertions(+), 8 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index 22d4f253c3..68c2bcc6fe 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -489,10 +488,11 @@ public class BlockDeletingService extends BackgroundService {
Handler handler = Objects.requireNonNull(ozoneContainer.getDispatcher()
.getHandler(container.getContainerType()));
- Pair<Integer, Long> deleteBlocksResult =
+ DeleteTransactionStats deleteBlocksResult =
deleteTransactions(delBlocks, handler, blockDataTable, container);
- int deletedBlocksCount = deleteBlocksResult.getLeft();
- long releasedBytes = deleteBlocksResult.getRight();
+ int deletedBlocksProcessed = deleteBlocksResult.getBlocksProcessed();
+ int deletedBlocksCount = deleteBlocksResult.getBlocksDeleted();
+ long releasedBytes = deleteBlocksResult.getBytesReleased();
// Once blocks are deleted... remove the blockID from blockDataTable
// and also remove the transactions from txnTable.
@@ -516,7 +516,7 @@ public class BlockDeletingService extends BackgroundService {
// update count of pending deletion blocks, block count and used
// bytes in in-memory container status and used space in volume.
- containerData.decrPendingDeletionBlocks(deletedBlocksCount);
+ containerData.decrPendingDeletionBlocks(deletedBlocksProcessed);
containerData.decrBlockCount(deletedBlocksCount);
containerData.decrBytesUsed(releasedBytes);
containerData.getVolume().decrementUsedSpace(releasedBytes);
@@ -541,10 +541,11 @@ public class BlockDeletingService extends BackgroundService {
* Delete the chunks for the given blocks.
* Return the deletedBlocks count and number of bytes released.
*/
- private Pair<Integer, Long> deleteTransactions(
+ private DeleteTransactionStats deleteTransactions(
List<DeletedBlocksTransaction> delBlocks, Handler handler,
Table<String, BlockData> blockDataTable, Container container)
throws IOException {
+ int blocksProcessed = 0;
int blocksDeleted = 0;
long bytesReleased = 0;
for (DeletedBlocksTransaction entry : delBlocks) {
@@ -556,7 +557,7 @@ public class BlockDeletingService extends BackgroundService {
LOG.warn("Missing delete block(Container = " +
container.getContainerData().getContainerID() + ", Block = " +
blkLong);
- blocksDeleted++;
+ blocksProcessed++;
continue;
}
@@ -564,6 +565,7 @@ public class BlockDeletingService extends BackgroundService {
try {
handler.deleteBlock(container, blkInfo);
blocksDeleted++;
+ blocksProcessed++;
deleted = true;
} catch (IOException e) {
// TODO: if deletion of certain block retries exceed the certain
@@ -584,7 +586,8 @@ public class BlockDeletingService extends BackgroundService {
}
}
}
- return Pair.of(blocksDeleted, bytesReleased);
+ return new DeleteTransactionStats(blocksProcessed,
+ blocksDeleted, bytesReleased);
}
@Override
@@ -601,4 +604,32 @@ public class BlockDeletingService extends BackgroundService {
public BlockDeletingServiceMetrics getMetrics() {
return metrics;
}
+
+ /**
+ * The wrapper class of the result of deleting transactions.
+ */
+ private static class DeleteTransactionStats {
+
+ private final int blocksProcessed;
+ private final int blocksDeleted;
+ private final long bytesReleased;
+
+ DeleteTransactionStats(int proceeded, int deleted, long released) {
+ blocksProcessed = proceeded;
+ blocksDeleted = deleted;
+ bytesReleased = released;
+ }
+
+ public int getBlocksProcessed() {
+ return blocksProcessed;
+ }
+
+ public int getBlocksDeleted() {
+ return blocksDeleted;
+ }
+
+ public long getBytesReleased() {
+ return bytesReleased;
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 4ae420dc72..fe39959688 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -558,6 +558,12 @@ public class TestBlockDeletingService {
GenericTestUtils.waitFor(() -> ctr2.getNumPendingDeletionBlocks() == 0,
200, 2000);
+ // To make sure the container stat calculation is right
+ Assert.assertEquals(0, ctr1.getBlockCount());
+ Assert.assertEquals(0, ctr1.getBytesUsed());
+ Assert.assertEquals(0, ctr2.getBlockCount());
+ Assert.assertEquals(0, ctr2.getBytesUsed());
+
svc.shutdown();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org