You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ri...@apache.org on 2023/02/06 05:01:13 UTC
[ozone] branch master updated: HDDS-7126. Clean deletedBlock records of deleted containers (#3683)
This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new db1ea10645 HDDS-7126. Clean deletedBlock records of deleted containers (#3683)
db1ea10645 is described below
commit db1ea10645b7ec8b4aa23c7efef2c30ac4919025
Author: Symious <14...@users.noreply.github.com>
AuthorDate: Mon Feb 6 13:01:07 2023 +0800
HDDS-7126. Clean deletedBlock records of deleted containers (#3683)
=
---
.../hadoop/hdds/scm/container/ContainerInfo.java | 3 +++
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 8 ++++++-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 28 +++++++++++++++++++---
3 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
index e99b6a5c10..74373e1e54 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
@@ -506,4 +506,7 @@ public class ContainerInfo implements Comparator<ContainerInfo>,
|| state == HddsProtos.LifeCycleState.CLOSING;
}
+ public boolean isDeleted() {
+ return state == HddsProtos.LifeCycleState.DELETED;
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index 31877876e1..6903d5ed3f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -436,7 +436,13 @@ public class DeletedBlockLogImpl
DeletedBlocksTransaction txn = keyValue.getValue();
final ContainerID id = ContainerID.valueOf(txn.getContainerID());
try {
- if (txn.getCount() > -1 && txn.getCount() <= maxRetry
+ // HDDS-7126. When container is under replicated, it is possible
+ // that container is deleted, but transactions are not deleted.
+ if (containerManager.getContainer(id).isDeleted()) {
+ LOG.warn("Container: " + id + " was deleted for the " +
+ "transaction: " + txn);
+ txIDs.add(txn.getTxID());
+ } else if (txn.getCount() > -1 && txn.getCount() <= maxRetry
&& !containerManager.getContainer(id).isOpen()) {
getTransaction(txn, transactions);
transactionToDNsCommitMap
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
index 7c2f2ddebb..033c1a0745 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
@@ -162,13 +162,14 @@ public class TestDeletedBlockLog {
}).when(containerManager).updateDeleteTransactionId(anyObject());
}
- private void updateContainerMetadata(long cid) throws IOException {
+ private void updateContainerMetadata(long cid,
+ HddsProtos.LifeCycleState state) throws IOException {
final ContainerInfo container =
new ContainerInfo.Builder()
.setContainerID(cid)
.setReplicationConfig(RatisReplicationConfig.getInstance(
ReplicationFactor.THREE))
- .setState(HddsProtos.LifeCycleState.CLOSED)
+ .setState(state)
.setOwner("TestDeletedBlockLog")
.setPipelineID(PipelineID.randomId())
.build();
@@ -193,13 +194,18 @@ public class TestDeletedBlockLog {
}
private Map<Long, List<Long>> generateData(int dataSize) throws IOException {
+ return generateData(dataSize, HddsProtos.LifeCycleState.CLOSED);
+ }
+
+ private Map<Long, List<Long>> generateData(int dataSize,
+ HddsProtos.LifeCycleState state) throws IOException {
Map<Long, List<Long>> blockMap = new HashMap<>();
Random random = new Random(1);
int continerIDBase = random.nextInt(100);
int localIDBase = random.nextInt(1000);
for (int i = 0; i < dataSize; i++) {
long containerID = continerIDBase + i;
- updateContainerMetadata(containerID);
+ updateContainerMetadata(containerID, state);
List<Long> blocks = new ArrayList<>();
for (int j = 0; j < BLOCKS_PER_TXN; j++) {
long localID = localIDBase + j;
@@ -580,6 +586,22 @@ public class TestDeletedBlockLog {
Assertions.assertEquals(2, blocks.size());
}
+ @Test
+ public void testDeletedBlockTransactionsOfDeletedContainer()
+ throws IOException, TimeoutException {
+ int txNum = 10;
+ List<DeletedBlocksTransaction> blocks;
+
+ // Creates {TXNum} TX in the log.
+ Map<Long, List<Long>> deletedBlocks = generateData(txNum,
+ HddsProtos.LifeCycleState.DELETED);
+ addTransactions(deletedBlocks, true);
+
+ blocks = getTransactions(txNum * BLOCKS_PER_TXN);
+ // There should be no txn remaining
+ Assertions.assertEquals(0, blocks.size());
+ }
+
private void mockStandAloneContainerInfo(long containerID, DatanodeDetails dd)
throws IOException {
List<DatanodeDetails> dns = Collections.singletonList(dd);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org