You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2023/04/05 13:18:11 UTC
[ozone] branch master updated: HDDS-8039. Allow container inspector to run from ozone debug. (#4337)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d4d90c7d1a HDDS-8039. Allow container inspector to run from ozone debug. (#4337)
d4d90c7d1a is described below
commit d4d90c7d1acbf3c77f50b342e138f9c8fde12039
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Wed Apr 5 21:18:03 2023 +0800
HDDS-8039. Allow container inspector to run from ozone debug. (#4337)
---
.../ozone/container/common/impl/ContainerSet.java | 7 +-
.../KeyValueContainerMetadataInspector.java | 157 +++++++++++++---
.../container/ozoneimpl/ContainerController.java | 4 +
.../keyvalue/ContainerTestVersionInfo.java | 5 +
.../TestKeyValueContainerIntegrityChecks.java | 3 +-
.../TestKeyValueContainerMetadataInspector.java | 197 +++++++++++++++++++--
.../ozone/debug/container/ContainerCommands.java | 5 +
.../ozone/debug/container/InspectSubcommand.java | 74 ++++++++
8 files changed, 412 insertions(+), 40 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index 0c5ae6aeeb..2f4eaf5ec7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -48,7 +48,7 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Con
/**
* Class that manages Containers created on the datanode.
*/
-public class ContainerSet {
+public class ContainerSet implements Iterable<Container<?>> {
private static final Logger LOG = LoggerFactory.getLogger(ContainerSet.class);
@@ -201,6 +201,11 @@ public class ContainerSet {
* @return {@literal Iterator<Container<?>>}
*/
public Iterator<Container<?>> getContainerIterator() {
+ return iterator();
+ }
+
+ @Override
+ public Iterator<Container<?>> iterator() {
return containerMap.values().iterator();
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
index 595aa925a4..6a41416bba 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
@@ -102,6 +102,10 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
private Mode mode;
+ public KeyValueContainerMetadataInspector(Mode mode) {
+ this.mode = mode;
+ }
+
public KeyValueContainerMetadataInspector() {
mode = Mode.OFF;
}
@@ -155,10 +159,15 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
@Override
public void process(ContainerData containerData, DatanodeStore store) {
+ process(containerData, store, REPORT_LOG);
+ }
+
+ public String process(ContainerData containerData, DatanodeStore store,
+ Logger log) {
// If the system property to process container metadata was not
// specified, or the inspector is unloaded, this method is a no-op.
if (mode == Mode.OFF) {
- return;
+ return null;
}
KeyValueContainerData kvData = null;
@@ -167,7 +176,7 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
} else {
LOG.error("This inspector only works on KeyValueContainers. Inspection " +
"will not be run for container {}", containerData.getContainerID());
- return;
+ return null;
}
JsonObject containerJson = inspectContainer(kvData, store);
@@ -178,14 +187,17 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
.serializeNulls()
.create();
String jsonReport = gson.toJson(containerJson);
- if (correct) {
- REPORT_LOG.trace(jsonReport);
- } else {
- REPORT_LOG.error(jsonReport);
+ if (log != null) {
+ if (correct) {
+ log.trace(jsonReport);
+ } else {
+ log.error(jsonReport);
+ }
}
+ return jsonReport;
}
- private JsonObject inspectContainer(KeyValueContainerData containerData,
+ static JsonObject inspectContainer(KeyValueContainerData containerData,
DatanodeStore store) {
JsonObject containerJson = new JsonObject();
@@ -224,7 +236,7 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
return containerJson;
}
- private JsonObject getDBMetadataJson(Table<String, Long> metadataTable,
+ static JsonObject getDBMetadataJson(Table<String, Long> metadataTable,
KeyValueContainerData containerData) throws IOException {
JsonObject dBMetadata = new JsonObject();
@@ -242,14 +254,13 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
return dBMetadata;
}
- private JsonObject getAggregateValues(DatanodeStore store,
+ static JsonObject getAggregateValues(DatanodeStore store,
KeyValueContainerData containerData, String schemaVersion)
throws IOException {
JsonObject aggregates = new JsonObject();
long usedBytesTotal = 0;
long blockCountTotal = 0;
- long pendingDeleteBlockCountTotal = 0;
// Count normal blocks.
try (BlockIterator<BlockData> blockIter =
store.getBlockIterator(containerData.getContainerID(),
@@ -262,7 +273,10 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
}
// Count pending delete blocks.
+ final PendingDelete pendingDelete;
if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
+ long pendingDeleteBlockCountTotal = 0;
+ long pendingDeleteBytes = 0;
try (BlockIterator<BlockData> blockIter =
store.getBlockIterator(containerData.getContainerID(),
containerData.getDeletingBlockKeyFilter())) {
@@ -270,18 +284,22 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
while (blockIter.hasNext()) {
blockCountTotal++;
pendingDeleteBlockCountTotal++;
- usedBytesTotal += getBlockLength(blockIter.nextBlock());
+ final long bytes = getBlockLength(blockIter.nextBlock());
+ usedBytesTotal += bytes;
+ pendingDeleteBytes += bytes;
}
}
+ pendingDelete = new PendingDelete(
+ pendingDeleteBlockCountTotal, pendingDeleteBytes);
} else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) {
DatanodeStoreSchemaTwoImpl schemaTwoStore =
(DatanodeStoreSchemaTwoImpl) store;
- pendingDeleteBlockCountTotal =
- countPendingDeletesSchemaV2(schemaTwoStore);
+ pendingDelete =
+ countPendingDeletesSchemaV2(schemaTwoStore, containerData);
} else if (schemaVersion.equals(OzoneConsts.SCHEMA_V3)) {
DatanodeStoreSchemaThreeImpl schemaThreeStore =
(DatanodeStoreSchemaThreeImpl) store;
- pendingDeleteBlockCountTotal =
+ pendingDelete =
countPendingDeletesSchemaV3(schemaThreeStore, containerData);
} else {
throw new IOException("Failed to process deleted blocks for unknown " +
@@ -290,13 +308,12 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
aggregates.addProperty("blockCount", blockCountTotal);
aggregates.addProperty("usedBytes", usedBytesTotal);
- aggregates.addProperty("pendingDeleteBlocks",
- pendingDeleteBlockCountTotal);
+ pendingDelete.addToJson(aggregates);
return aggregates;
}
- private JsonObject getChunksDirectoryJson(File chunksDir) throws IOException {
+ static JsonObject getChunksDirectoryJson(File chunksDir) throws IOException {
JsonObject chunksDirectory = new JsonObject();
chunksDirectory.addProperty("path", chunksDir.getAbsolutePath());
@@ -321,6 +338,9 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
Table<String, Long> metadataTable = store.getMetadataTable();
+ final JsonObject dBMetadata = parent.getAsJsonObject("dBMetadata");
+ final JsonObject aggregates = parent.getAsJsonObject("aggregates");
+
// Check and repair block count.
JsonElement blockCountDB = parent.getAsJsonObject("dBMetadata")
.get(OzoneConsts.BLOCK_COUNT);
@@ -392,6 +412,36 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
errors.add(usedBytesError);
}
+ // check and repair if db delete count mismatches delete transaction count.
+ final JsonElement pendingDeleteCountDB = dBMetadata.get(
+ OzoneConsts.PENDING_DELETE_BLOCK_COUNT);
+ final long dbDeleteCount = jsonToLong(pendingDeleteCountDB);
+ final JsonElement pendingDeleteCountAggregate
+ = aggregates.get(PendingDelete.COUNT);
+ final long deleteTransactionCount = jsonToLong(pendingDeleteCountAggregate);
+ if (dbDeleteCount != deleteTransactionCount) {
+ passed = false;
+
+ final BooleanSupplier deleteCountRepairAction = () -> {
+ final String key = containerData.getPendingDeleteBlockCountKey();
+ try {
+ // set delete block count metadata table to delete transaction count
+ metadataTable.put(key, deleteTransactionCount);
+ return true;
+ } catch (IOException ex) {
+ LOG.error("Failed to reset {} for container {}.",
+ key, containerData.getContainerID(), ex);
+ }
+ return false;
+ };
+
+ final JsonObject deleteCountError = buildErrorAndRepair(
+ "dBMetadata." + OzoneConsts.PENDING_DELETE_BLOCK_COUNT,
+ pendingDeleteCountAggregate, pendingDeleteCountDB,
+ deleteCountRepairAction);
+ errors.add(deleteCountError);
+ }
+
// check and repair chunks dir.
JsonElement chunksDirPresent = parent.getAsJsonObject("chunksDirectory")
.get("present");
@@ -421,6 +471,10 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
return passed;
}
+ static long jsonToLong(JsonElement e) {
+ return e == null || e.isJsonNull() ? 0 : e.getAsLong();
+ }
+
private JsonObject buildErrorAndRepair(String property, JsonElement expected,
JsonElement actual, BooleanSupplier repairAction) {
JsonObject error = new JsonObject();
@@ -437,30 +491,81 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
return error;
}
- private long countPendingDeletesSchemaV2(DatanodeStoreSchemaTwoImpl
- schemaTwoStore) throws IOException {
+ static class PendingDelete {
+ static final String COUNT = "pendingDeleteBlocks";
+ static final String BYTES = "pendingDeleteBytes";
+
+ private final long count;
+ private final long bytes;
+
+ PendingDelete(long count, long bytes) {
+ this.count = count;
+ this.bytes = bytes;
+ }
+
+ void addToJson(JsonObject json) {
+ json.addProperty(COUNT, count);
+ json.addProperty(BYTES, bytes);
+ }
+ }
+
+ static PendingDelete countPendingDeletesSchemaV2(
+ DatanodeStoreSchemaTwoImpl schemaTwoStore,
+ KeyValueContainerData containerData) throws IOException {
long pendingDeleteBlockCountTotal = 0;
+ long pendingDeleteBytes = 0;
+
Table<Long, DeletedBlocksTransaction> delTxTable =
schemaTwoStore.getDeleteTransactionTable();
+ final Table<String, BlockData> blockDataTable
+ = schemaTwoStore.getBlockDataTable();
+
try (TableIterator<Long, ? extends Table.KeyValue<Long,
DeletedBlocksTransaction>> iterator = delTxTable.iterator()) {
while (iterator.hasNext()) {
DeletedBlocksTransaction txn = iterator.next().getValue();
+ final List<Long> localIDs = txn.getLocalIDList();
// In schema 2, pending delete blocks are stored in the
// transaction object. Since the actual blocks still exist in the
// block data table with no prefix, they have already been
// counted towards bytes used and total block count above.
- pendingDeleteBlockCountTotal += txn.getLocalIDList().size();
+ pendingDeleteBlockCountTotal += localIDs.size();
+ pendingDeleteBytes += computePendingDeleteBytes(
+ localIDs, containerData, blockDataTable);
}
}
- return pendingDeleteBlockCountTotal;
+ return new PendingDelete(pendingDeleteBlockCountTotal,
+ pendingDeleteBytes);
+ }
+
+ static long computePendingDeleteBytes(List<Long> localIDs,
+ KeyValueContainerData containerData,
+ Table<String, BlockData> blockDataTable) {
+ long pendingDeleteBytes = 0;
+ for (long id : localIDs) {
+ try {
+ final String blockKey = containerData.getBlockKey(id);
+ final BlockData blockData = blockDataTable.get(blockKey);
+ if (blockData != null) {
+ pendingDeleteBytes += blockData.getSize();
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to get block " + id
+ + " in container " + containerData.getContainerID()
+ + " from blockDataTable", e);
+ }
+ }
+ return pendingDeleteBytes;
}
- private long countPendingDeletesSchemaV3(
+ static PendingDelete countPendingDeletesSchemaV3(
DatanodeStoreSchemaThreeImpl schemaThreeStore,
KeyValueContainerData containerData) throws IOException {
long pendingDeleteBlockCountTotal = 0;
+ long pendingDeleteBytes = 0;
+ final Table<String, BlockData> blockDataTable
+ = schemaThreeStore.getBlockDataTable();
try (
TableIterator<String, ? extends Table.KeyValue<String,
DeletedBlocksTransaction>>
@@ -468,10 +573,14 @@ public class KeyValueContainerMetadataInspector implements ContainerInspector {
.iterator(containerData.containerPrefix())) {
while (iter.hasNext()) {
DeletedBlocksTransaction delTx = iter.next().getValue();
- pendingDeleteBlockCountTotal += delTx.getLocalIDList().size();
+ final List<Long> localIDs = delTx.getLocalIDList();
+ pendingDeleteBlockCountTotal += localIDs.size();
+ pendingDeleteBytes += computePendingDeleteBytes(
+ localIDs, containerData, blockDataTable);
}
- return pendingDeleteBlockCountTotal;
}
+ return new PendingDelete(pendingDeleteBlockCountTotal,
+ pendingDeleteBytes);
}
private static long getBlockLength(BlockData block) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
index 4087483d72..683aa0b053 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
@@ -196,6 +196,10 @@ public class ContainerController {
return containerSet.getContainerIterator();
}
+ public Iterable<Container<?>> getContainerSet() {
+ return containerSet;
+ }
+
/**
* Return an iterator of containers which are associated with the specified
* <code>volume</code>.
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
index c1292ea46c..61487f18b9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
@@ -65,6 +65,11 @@ public class ContainerTestVersionInfo {
.collect(toList());
}
+ @Override
+ public String toString() {
+ return "schema=" + schemaVersion + ", layout=" + layout;
+ }
+
public static List<ContainerTestVersionInfo> getLayoutList() {
return layoutList;
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
index cf18fa8948..49dc2e9554 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
@@ -58,7 +58,7 @@ import static org.junit.Assert.assertNotNull;
*/
public class TestKeyValueContainerIntegrityChecks {
- private static final Logger LOG =
+ static final Logger LOG =
LoggerFactory.getLogger(TestKeyValueContainerIntegrityChecks.class);
private final ContainerLayoutTestInfo containerLayoutTestInfo;
@@ -75,6 +75,7 @@ public class TestKeyValueContainerIntegrityChecks {
public TestKeyValueContainerIntegrityChecks(
ContainerTestVersionInfo versionInfo) {
+ LOG.info("new {} for {}", getClass().getSimpleName(), versionInfo);
this.conf = new OzoneConfiguration();
ContainerTestVersionInfo.setTestSchemaVersion(
versionInfo.getSchemaVersion(), conf);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
index aea451bc3a..f201bf5b48 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
@@ -22,6 +22,8 @@ import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerInspector;
@@ -29,6 +31,9 @@ import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
import org.apache.hadoop.ozone.container.common.utils.ContainerInspectorUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
+import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
+import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
+import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl;
import org.apache.log4j.PatternLayout;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.Assert;
@@ -36,6 +41,11 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
/**
* Tests for {@link KeyValueContainerMetadataInspector}.
*/
@@ -113,7 +123,7 @@ public class TestKeyValueContainerMetadataInspector
KeyValueContainer container = createClosedContainer(createBlocks);
setDBBlockAndByteCounts(container.getContainerData(), setBlocks, setBytes);
inspectThenRepairOnIncorrectContainer(container.getContainerData(),
- createBlocks, setBlocks, setBytes);
+ createBlocks, setBlocks, setBytes, 0, 0);
}
@Test
@@ -126,7 +136,7 @@ public class TestKeyValueContainerMetadataInspector
KeyValueContainer container = createOpenContainer(createBlocks);
setDBBlockAndByteCounts(container.getContainerData(), setBlocks, setBytes);
inspectThenRepairOnIncorrectContainer(container.getContainerData(),
- createBlocks, setBlocks, setBytes);
+ createBlocks, setBlocks, setBytes, 0, 0);
}
@Test
@@ -151,6 +161,99 @@ public class TestKeyValueContainerMetadataInspector
inspectThenRepairOnCorrectContainer(container.getContainerData());
}
+ static class DeletedBlocksTransactionGeneratorForTesting {
+ private long txId = 100;
+ private long localId = 2000;
+
+ DeletedBlocksTransaction next(long containerId, int numBlocks) {
+ final DeletedBlocksTransaction.Builder b
+ = DeletedBlocksTransaction.newBuilder()
+ .setContainerID(containerId)
+ .setTxID(txId++)
+ .setCount(0);
+ for (int i = 0; i < numBlocks; i++) {
+ b.addLocalID(localId++);
+ }
+ return b.build();
+ }
+
+ List<DeletedBlocksTransaction> generate(
+ long containerId, List<Integer> numBlocks) {
+ final List<DeletedBlocksTransaction> transactions = new ArrayList<>();
+ for (int n : numBlocks) {
+ transactions.add(next(containerId, n));
+ }
+ return transactions;
+ }
+ }
+
+ static final DeletedBlocksTransactionGeneratorForTesting GENERATOR
+ = new DeletedBlocksTransactionGeneratorForTesting();
+
+ @Test
+ public void testCorrectDeleteWithTransaction() throws Exception {
+ final int createBlocks = 4;
+ final int setBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createBlocks;
+ final int deleteCount = 10;
+
+ final KeyValueContainer container = createClosedContainer(createBlocks);
+ final List<DeletedBlocksTransaction> deleteTransactions
+ = GENERATOR.generate(container.getContainerData().getContainerID(),
+ Arrays.asList(1, 6, 3));
+ final long numDeletedLocalIds = deleteTransactions.stream()
+ .mapToLong(DeletedBlocksTransaction::getLocalIDCount).sum();
+ LOG.info("deleteTransactions = {}", deleteTransactions);
+ LOG.info("numDeletedLocalIds = {}", numDeletedLocalIds);
+ Assert.assertEquals(deleteCount, numDeletedLocalIds);
+
+ setDB(container.getContainerData(), createBlocks,
+ setBytes, deleteCount, deleteTransactions);
+ inspectThenRepairOnCorrectContainer(container.getContainerData());
+ }
+
+ @Test
+ public void testIncorrectDeleteWithTransaction() throws Exception {
+ final int createBlocks = 4;
+ final int setBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createBlocks;
+ final int deleteCount = 10;
+
+ final KeyValueContainer container = createClosedContainer(createBlocks);
+ final List<DeletedBlocksTransaction> deleteTransactions
+ = GENERATOR.generate(container.getContainerData().getContainerID(),
+ Arrays.asList(1, 3));
+ final long numDeletedLocalIds = deleteTransactions.stream()
+ .mapToLong(DeletedBlocksTransaction::getLocalIDCount).sum();
+ LOG.info("deleteTransactions = {}", deleteTransactions);
+ LOG.info("numDeletedLocalIds = {}", numDeletedLocalIds);
+
+ setDB(container.getContainerData(), createBlocks,
+ setBytes, deleteCount, deleteTransactions);
+ inspectThenRepairOnIncorrectContainer(container.getContainerData(),
+ createBlocks, createBlocks, setBytes,
+ deleteCount, numDeletedLocalIds);
+ }
+
+ @Test
+ public void testIncorrectDeleteWithoutTransaction() throws Exception {
+ final int createBlocks = 4;
+ final int setBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createBlocks;
+ final int deleteCount = 10;
+
+ final KeyValueContainer container = createClosedContainer(createBlocks);
+ final List<DeletedBlocksTransaction> deleteTransactions
+ = Collections.emptyList();
+ final long numDeletedLocalIds = deleteTransactions.stream()
+ .mapToLong(DeletedBlocksTransaction::getLocalIDCount).sum();
+ LOG.info("deleteTransactions = {}", deleteTransactions);
+ LOG.info("numDeletedLocalIds = {}", numDeletedLocalIds);
+
+ setDB(container.getContainerData(), createBlocks,
+ setBytes, deleteCount, deleteTransactions);
+ inspectThenRepairOnIncorrectContainer(container.getContainerData(),
+ createBlocks, createBlocks, setBytes,
+ deleteCount, numDeletedLocalIds);
+ }
+
public void inspectThenRepairOnCorrectContainer(
KeyValueContainerData containerData) throws Exception {
// No output for correct containers.
@@ -169,10 +272,14 @@ public class TestKeyValueContainerMetadataInspector
* @param createdBlocks Number of blocks to create in the container.
* @param setBlocks total block count value set in the database.
* @param setBytes total used bytes value set in the database.
+ * @param deleteCount total deleted block count value set in the database.
+ * @param numDeletedLocalIds total number of deleted block local id count
+ * in the transactions
*/
public void inspectThenRepairOnIncorrectContainer(
KeyValueContainerData containerData, int createdBlocks, int setBlocks,
- int setBytes) throws Exception {
+ int setBytes, int deleteCount, long numDeletedLocalIds)
+ throws Exception {
int createdBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createdBlocks;
int createdFiles = 0;
switch (getChunkLayout()) {
@@ -194,24 +301,26 @@ public class TestKeyValueContainerMetadataInspector
checkJsonReportForIncorrectContainer(inspectJson,
containerState, createdBlocks, setBlocks, createdBytes, setBytes,
- createdFiles, false);
+ createdFiles, deleteCount, numDeletedLocalIds, false);
// Container should not have been modified in inspect mode.
- checkDBBlockAndByteCounts(containerData, setBlocks, setBytes);
+ checkDbCounts(containerData, setBlocks, setBytes, deleteCount);
// Now repair the container.
JsonObject repairJson = runInspectorAndGetReport(containerData,
KeyValueContainerMetadataInspector.Mode.REPAIR);
checkJsonReportForIncorrectContainer(repairJson,
containerState, createdBlocks, setBlocks, createdBytes, setBytes,
- createdFiles, true);
+ createdFiles, deleteCount, numDeletedLocalIds, true);
// Metadata keys should have been fixed.
- checkDBBlockAndByteCounts(containerData, createdBlocks, createdBytes);
+ checkDbCounts(containerData, createdBlocks, createdBytes,
+ numDeletedLocalIds);
}
@SuppressWarnings("checkstyle:ParameterNumber")
private void checkJsonReportForIncorrectContainer(JsonObject inspectJson,
String expectedContainerState, long createdBlocks,
long setBlocks, long createdBytes, long setBytes, long createdFiles,
+ long setPendingDeleteCount, long createdPendingDeleteCount,
boolean shouldRepair) {
// Check main container properties.
Assert.assertEquals(inspectJson.get("containerID").getAsLong(),
@@ -232,7 +341,7 @@ public class TestKeyValueContainerMetadataInspector
jsonAggregates.get("blockCount").getAsLong());
Assert.assertEquals(createdBytes,
jsonAggregates.get("usedBytes").getAsLong());
- Assert.assertEquals(0,
+ Assert.assertEquals(createdPendingDeleteCount,
jsonAggregates.get("pendingDeleteBlocks").getAsLong());
// Check chunks directory.
@@ -243,11 +352,24 @@ public class TestKeyValueContainerMetadataInspector
// Check errors.
checkJsonErrorsReport(inspectJson, "dBMetadata.#BLOCKCOUNT",
- new JsonPrimitive(createdBlocks), new JsonPrimitive(setBlocks),
- shouldRepair);
+ createdBlocks, setBlocks, shouldRepair);
checkJsonErrorsReport(inspectJson, "dBMetadata.#BYTESUSED",
- new JsonPrimitive(createdBytes), new JsonPrimitive(setBytes),
- shouldRepair);
+ createdBytes, setBytes, shouldRepair);
+ checkJsonErrorsReport(inspectJson, "dBMetadata.#PENDINGDELETEBLOCKCOUNT",
+ createdPendingDeleteCount, setPendingDeleteCount, shouldRepair);
+ }
+
+ private void checkJsonErrorsReport(
+ JsonObject jsonReport, String propertyValue,
+ long correctExpected, long correctActual,
+ boolean correctRepair) {
+ if (correctExpected == correctActual) {
+ return;
+ }
+ checkJsonErrorsReport(jsonReport, propertyValue,
+ new JsonPrimitive(correctExpected),
+ new JsonPrimitive(correctActual),
+ correctRepair);
}
/**
@@ -290,16 +412,59 @@ public class TestKeyValueContainerMetadataInspector
public void setDBBlockAndByteCounts(KeyValueContainerData containerData,
long blockCount, long byteCount) throws Exception {
+ setDB(containerData, blockCount, byteCount,
+ 0, Collections.emptyList());
+ }
+
+ public void setDB(KeyValueContainerData containerData,
+ long blockCount, long byteCount,
+ long dbDeleteCount, List<DeletedBlocksTransaction> deleteTransactions)
+ throws Exception {
try (DBHandle db = BlockUtils.getDB(containerData, getConf())) {
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
// Don't care about in memory state. Just change the DB values.
metadataTable.put(containerData.getBlockCountKey(), blockCount);
metadataTable.put(containerData.getBytesUsedKey(), byteCount);
+ metadataTable.put(containerData.getPendingDeleteBlockCountKey(),
+ dbDeleteCount);
+
+ final DatanodeStore store = db.getStore();
+ LOG.info("store {}", store.getClass().getSimpleName());
+ if (store instanceof DatanodeStoreSchemaTwoImpl) {
+ final DatanodeStoreSchemaTwoImpl s2store
+ = (DatanodeStoreSchemaTwoImpl)store;
+ final Table<Long, DeletedBlocksTransaction> delTxTable
+ = s2store.getDeleteTransactionTable();
+ try (BatchOperation batch = store.getBatchHandler()
+ .initBatchOperation()) {
+ for (DeletedBlocksTransaction t : deleteTransactions) {
+ delTxTable.putWithBatch(batch, t.getTxID(), t);
+ }
+ store.getBatchHandler().commitBatchOperation(batch);
+ }
+ } else if (store instanceof DatanodeStoreSchemaThreeImpl) {
+ final DatanodeStoreSchemaThreeImpl s3store
+ = (DatanodeStoreSchemaThreeImpl)store;
+ final Table<String, DeletedBlocksTransaction> delTxTable
+ = s3store.getDeleteTransactionTable();
+ try (BatchOperation batch = store.getBatchHandler()
+ .initBatchOperation()) {
+ for (DeletedBlocksTransaction t : deleteTransactions) {
+ final String key = containerData.getDeleteTxnKey(t.getTxID());
+ delTxTable.putWithBatch(batch, key, t);
+ }
+ store.getBatchHandler().commitBatchOperation(batch);
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ "Unsupported store class " + store.getClass().getSimpleName());
+ }
}
}
- public void checkDBBlockAndByteCounts(KeyValueContainerData containerData,
- long expectedBlockCount, long expectedBytesUsed) throws Exception {
+ void checkDbCounts(KeyValueContainerData containerData,
+ long expectedBlockCount, long expectedBytesUsed,
+ long expectedDeletedCount) throws Exception {
try (DBHandle db = BlockUtils.getDB(containerData, getConf())) {
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
@@ -308,6 +473,10 @@ public class TestKeyValueContainerMetadataInspector
long blockCount = metadataTable.get(containerData.getBlockCountKey());
Assert.assertEquals(expectedBlockCount, blockCount);
+
+ final long deleteCount = metadataTable.get(
+ containerData.getPendingDeleteBlockCountKey());
+ Assert.assertEquals(expectedDeletedCount, deleteCount);
}
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
index ea7b10216c..5592926bf8 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
@@ -79,6 +79,7 @@ import java.util.stream.Stream;
ListSubcommand.class,
InfoSubcommand.class,
ExportSubcommand.class,
+ InspectSubcommand.class
})
@MetaInfServices(SubcommandWithParent.class)
public class ContainerCommands implements Callable<Void>, SubcommandWithParent {
@@ -107,6 +108,10 @@ public class ContainerCommands implements Callable<Void>, SubcommandWithParent {
return OzoneDebug.class;
}
+ OzoneConfiguration getOzoneConf() {
+ return parent.getOzoneConf();
+ }
+
public void loadContainersFromVolumes() throws IOException {
OzoneConfiguration conf = parent.getOzoneConf();
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java
new file mode 100644
index 0000000000..86f5d54e20
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug.container;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerMetadataInspector;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+/**
+ * {@code ozone debug container inspect},
+ * a command to run {@link KeyValueContainerMetadataInspector}.
+ */
+@Command(
+ name = "inspect",
+ description
+ = "Check the metadata of all container replicas on this datanode.")
+public class InspectSubcommand implements Callable<Void> {
+
+ @CommandLine.ParentCommand
+ private ContainerCommands parent;
+
+ @Override
+ public Void call() throws IOException {
+ final OzoneConfiguration conf = parent.getOzoneConf();
+ parent.loadContainersFromVolumes();
+
+ final KeyValueContainerMetadataInspector inspector
+ = new KeyValueContainerMetadataInspector(
+ KeyValueContainerMetadataInspector.Mode.INSPECT);
+ for (Container<?> container : parent.getController().getContainerSet()) {
+ final ContainerData data = container.getContainerData();
+ if (!(data instanceof KeyValueContainerData)) {
+ continue;
+ }
+ final KeyValueContainerData kvData = (KeyValueContainerData) data;
+ try (DatanodeStore store = BlockUtils.getUncachedDatanodeStore(
+ kvData, conf, true)) {
+ final String json = inspector.process(kvData, store, null);
+ System.out.println(json);
+ } catch (IOException e) {
+ System.err.print("Failed to inspect container "
+ + kvData.getContainerID() + ": ");
+ e.printStackTrace();
+ }
+ }
+
+ return null;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org