You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ha...@apache.org on 2020/08/11 21:43:42 UTC
[hadoop-ozone] branch master updated: HDDS-4061. Pending delete
blocks are not always included in #BLOCKCOUNT metadata (#1288)
This is an automated email from the ASF dual-hosted git repository.
hanishakoneru pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0f23b22 HDDS-4061. Pending delete blocks are not always included in #BLOCKCOUNT metadata (#1288)
0f23b22 is described below
commit 0f23b22fc90c62e02857cd805a5fff4af61d6e61
Author: Ethan Rose <33...@users.noreply.github.com>
AuthorDate: Tue Aug 11 17:43:31 2020 -0400
HDDS-4061. Pending delete blocks are not always included in #BLOCKCOUNT metadata (#1288)
---
.../keyvalue/helpers/KeyValueContainerUtil.java | 14 +++++++++++++-
.../ozone/container/ozoneimpl/TestContainerReader.java | 16 ++++------------
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index d5fea23..2958e9e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -230,10 +230,22 @@ public final class KeyValueContainerUtil {
private static void initializeUsedBytesAndBlockCount(
KeyValueContainerData kvContainerData) throws IOException {
+ MetadataKeyFilters.KeyPrefixFilter filter =
+ new MetadataKeyFilters.KeyPrefixFilter();
+
+ // Ignore all blocks except those with no prefix, or those with
+ // #deleting# prefix.
+ filter.addFilter(OzoneConsts.DELETED_KEY_PREFIX, true)
+ .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true)
+ .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true)
+ .addFilter(OzoneConsts.BLOCK_COUNT, true)
+ .addFilter(OzoneConsts.CONTAINER_BYTES_USED, true)
+ .addFilter(OzoneConsts.PENDING_DELETE_BLOCK_COUNT, true);
+
long blockCount = 0;
try (KeyValueBlockIterator blockIter = new KeyValueBlockIterator(
kvContainerData.getContainerID(),
- new File(kvContainerData.getContainerPath()))) {
+ new File(kvContainerData.getContainerPath()), filter)) {
long usedBytes = 0;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
index 15c0236..0014a75 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
@@ -52,7 +52,6 @@ import java.util.List;
import java.util.UUID;
import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY;
-import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_BYTES_USED_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -139,17 +138,10 @@ public class TestContainerReader {
}
if (setMetaData) {
+ // Pending delete blocks are still counted towards the block count
+ // and bytes used metadata values, so those do not change.
metadataStore.getStore().put(DB_PENDING_DELETE_BLOCK_COUNT_KEY,
Longs.toByteArray(count));
- long blkCount = Longs.fromByteArray(
- metadataStore.getStore().get(DB_BLOCK_COUNT_KEY));
- metadataStore.getStore().put(DB_BLOCK_COUNT_KEY,
- Longs.toByteArray(blkCount - count));
- long bytesUsed = Longs.fromByteArray(
- metadataStore.getStore().get(DB_CONTAINER_BYTES_USED_KEY));
- metadataStore.getStore().put(DB_CONTAINER_BYTES_USED_KEY,
- Longs.toByteArray(bytesUsed - (count * blockLen)));
-
}
}
@@ -210,10 +202,10 @@ public class TestContainerReader {
keyValueContainer.getContainerData();
// Verify block related metadata.
- Assert.assertEquals(blockCount - i,
+ Assert.assertEquals(blockCount,
keyValueContainerData.getKeyCount());
- Assert.assertEquals((blockCount - i) * blockLen,
+ Assert.assertEquals(blockCount * blockLen,
keyValueContainerData.getBytesUsed());
Assert.assertEquals(i,
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org