You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/10/22 15:25:02 UTC
(iotdb) 01/01: Clean walnode after drop database when using simple consensus
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch clean_walnode_simple_consensus
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5fb59ad4770822ab3b90ec59abdd7915d3e82d0e
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue Oct 22 23:24:39 2024 +0800
Clean walnode after drop database when using simple consensus
---
.../iotdb/db/storageengine/StorageEngine.java | 48 +++++++++++++---------
.../storageengine/dataregion/wal/WALManager.java | 21 ++++++++--
.../dataregion/wal/allocation/ElasticStrategy.java | 27 +++++++++++-
3 files changed, 72 insertions(+), 24 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index b75d8be26ae..67e72aacd65 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -758,28 +758,36 @@ public class StorageEngine implements IService {
region.abortCompaction();
region.syncDeleteDataFiles();
region.deleteFolder(systemDir);
- if (CONFIG.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)
- || CONFIG
- .getDataRegionConsensusProtocolClass()
- .equals(ConsensusFactory.IOT_CONSENSUS_V2)) {
- // delete wal
- WALManager.getInstance()
- .deleteWALNode(
- region.getDatabaseName() + FILE_NAME_SEPARATOR + region.getDataRegionId());
- // delete snapshot
- for (String dataDir : CONFIG.getLocalDataDirs()) {
- File regionSnapshotDir =
- new File(
- dataDir + File.separator + IoTDBConstant.SNAPSHOT_FOLDER_NAME,
- region.getDatabaseName() + FILE_NAME_SEPARATOR + regionId.getId());
- if (regionSnapshotDir.exists()) {
- try {
- FileUtils.deleteDirectory(regionSnapshotDir);
- } catch (IOException e) {
- LOGGER.error("Failed to delete snapshot dir {}", regionSnapshotDir, e);
+ switch (CONFIG.getDataRegionConsensusProtocolClass()) {
+ case ConsensusFactory.IOT_CONSENSUS:
+ case ConsensusFactory.IOT_CONSENSUS_V2:
+ // delete wal
+ WALManager.getInstance()
+ .deleteWALNode(
+ region.getDatabaseName() + FILE_NAME_SEPARATOR + region.getDataRegionId());
+ // delete snapshot
+ for (String dataDir : CONFIG.getLocalDataDirs()) {
+ File regionSnapshotDir =
+ new File(
+ dataDir + File.separator + IoTDBConstant.SNAPSHOT_FOLDER_NAME,
+ region.getDatabaseName() + FILE_NAME_SEPARATOR + regionId.getId());
+ if (regionSnapshotDir.exists()) {
+ try {
+ FileUtils.deleteDirectory(regionSnapshotDir);
+ } catch (IOException e) {
+ LOGGER.error("Failed to delete snapshot dir {}", regionSnapshotDir, e);
+ }
}
}
- }
+ break;
+ case ConsensusFactory.SIMPLE_CONSENSUS:
+ // remove region information in wal and may clean wal node
+ WALManager.getInstance()
+ .removeRegionAndCleanWalNode(region.getDatabaseName(), region.getDataRegionId());
+ break;
+ case ConsensusFactory.RATIS_CONSENSUS:
+ default:
+ break;
}
} catch (Exception e) {
LOGGER.error(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java
index 8b7f1f6336d..7fc7460fe8e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java
@@ -54,6 +54,8 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+
/** This class is used to manage and allocate wal nodes. */
public class WALManager implements IService {
private static final Logger logger = LoggerFactory.getLogger(WALManager.class);
@@ -79,13 +81,13 @@ public class WALManager implements IService {
}
}
- public static String getApplicantUniqueId(String storageGroupName, boolean sequence) {
+ public static String getApplicantUniqueId(String dataRegionName, boolean sequence) {
return config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)
|| config
.getDataRegionConsensusProtocolClass()
.equals(ConsensusFactory.IOT_CONSENSUS_V2)
- ? storageGroupName
- : storageGroupName
+ ? dataRegionName
+ : dataRegionName
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ (sequence ? "sequence" : "unsequence");
}
@@ -129,6 +131,19 @@ public class WALManager implements IService {
WritingMetrics.getInstance().removeWALNodeInfoMetrics(applicantUniqueId);
}
+ /** Region information will be removed only when using ElasticStrategy. */
+ public void removeRegionAndCleanWalNode(String databaseName, String dataRegionId) {
+ if (config.getWalMode() == WALMode.DISABLE || !(walNodesManager instanceof ElasticStrategy)) {
+ return;
+ }
+
+ String dataRegionName = databaseName + FILE_NAME_SEPARATOR + dataRegionId;
+ ((ElasticStrategy) walNodesManager)
+ .removeUniqueIdAndCleanWalNode(getApplicantUniqueId(dataRegionName, true));
+ ((ElasticStrategy) walNodesManager)
+ .removeUniqueIdAndCleanWalNode(getApplicantUniqueId(dataRegionName, false));
+ }
+
@Override
public void start() throws StartupException {
if (config.getWalMode() == WALMode.DISABLE) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/ElasticStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/ElasticStrategy.java
index 99007edc21e..a9feff6c981 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/ElasticStrategy.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/ElasticStrategy.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.allocation;
+import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.node.IWALNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
@@ -69,7 +71,7 @@ public class ElasticStrategy extends AbstractNodeAllocationStrategy {
}
walNodes.add((WALNode) node);
}
- uniqueId2Nodes.put(applicantUniqueId, walNodes.get(nodeIdCounter));
+ uniqueId2Nodes.put(applicantUniqueId, walNodes.get(walNodes.size() - 1));
}
return uniqueId2Nodes.get(applicantUniqueId);
@@ -78,6 +80,29 @@ public class ElasticStrategy extends AbstractNodeAllocationStrategy {
}
}
+ public void removeUniqueIdAndCleanWalNode(String applicantUniqueId) {
+ nodesLock.lock();
+ try {
+ WALNode walNode = uniqueId2Nodes.remove(applicantUniqueId);
+ if (!uniqueId2Nodes.containsValue(walNode)) {
+ if (walNode != null) {
+ walNode.close();
+ if (walNode.getLogDirectory().exists()) {
+ FileUtils.deleteFileOrDirectory(walNode.getLogDirectory());
+ }
+ WALManager.getInstance().subtractTotalDiskUsage(walNode.getDiskUsage());
+ WALManager.getInstance().subtractTotalFileNum(walNode.getFileNum());
+ }
+ walNodes.remove(walNode);
+ if (walNodes.isEmpty()) {
+ nodeIdCounter = -1;
+ }
+ }
+ } finally {
+ nodesLock.unlock();
+ }
+ }
+
@Override
public List<WALNode> getNodesSnapshot() {
List<WALNode> snapshot;