You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/07/29 02:07:31 UTC
[iotdb] branch master updated: add wal disk usage (#6822)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 52a92f6cd0 add wal disk usage (#6822)
52a92f6cd0 is described below
commit 52a92f6cd082aaee610907b0eda22bf458188ad1
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Fri Jul 29 10:07:26 2022 +0800
add wal disk usage (#6822)
---
.../main/java/org/apache/iotdb/db/wal/WALManager.java | 16 ++++++++++++++++
.../apache/iotdb/db/wal/buffer/AbstractWALBuffer.java | 2 ++
.../main/java/org/apache/iotdb/db/wal/node/WALNode.java | 3 +++
.../apache/iotdb/db/wal/recover/WALNodeRecoverTask.java | 6 ++++++
4 files changed, 27 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/WALManager.java b/server/src/main/java/org/apache/iotdb/db/wal/WALManager.java
index 6f6a789ec9..c1f27f3e71 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/WALManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/WALManager.java
@@ -44,6 +44,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
/** This class is used to manage and allocate wal nodes */
public class WALManager implements IService {
@@ -56,6 +57,8 @@ public class WALManager implements IService {
private final NodeAllocationStrategy walNodesManager;
/** single thread to delete old .wal files */
private ScheduledExecutorService walDeleteThread;
+ /** total disk usage of wal files */
+ private final AtomicLong totalDiskUsage = new AtomicLong();
private WALManager() {
if (config.isClusterMode()
@@ -164,6 +167,18 @@ public class WALManager implements IService {
}
}
+ public long getTotalDiskUsage() {
+ return totalDiskUsage.get();
+ }
+
+ public void addTotalDiskUsage(long size) {
+ totalDiskUsage.accumulateAndGet(size, Long::sum);
+ }
+
+ public void subtractTotalDiskUsage(long size) {
+ totalDiskUsage.accumulateAndGet(size, (x, y) -> x - y);
+ }
+
@Override
public void stop() {
if (config.getWalMode() == WALMode.DISABLE) {
@@ -198,6 +213,7 @@ public class WALManager implements IService {
@TestOnly
public void clear() {
+ totalDiskUsage.set(0);
walNodesManager.clear();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/buffer/AbstractWALBuffer.java b/server/src/main/java/org/apache/iotdb/db/wal/buffer/AbstractWALBuffer.java
index e73f053469..53b94089c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/buffer/AbstractWALBuffer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/buffer/AbstractWALBuffer.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.wal.buffer;
import org.apache.iotdb.commons.file.SystemFileFactory;
+import org.apache.iotdb.db.wal.WALManager;
import org.apache.iotdb.db.wal.io.WALWriter;
import org.apache.iotdb.db.wal.utils.WALFileStatus;
import org.apache.iotdb.db.wal.utils.WALFileUtils;
@@ -82,6 +83,7 @@ public abstract class AbstractWALBuffer implements IWALBuffer {
File currentFile = currentWALFileWriter.getLogFile();
String currentName = currentFile.getName();
currentWALFileWriter.close();
+ WALManager.getInstance().addTotalDiskUsage(currentWALFileWriter.size());
if (WALFileUtils.parseStatusCode(currentName) != fileStatus) {
String targetName =
WALFileUtils.getLogFileName(
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java b/server/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java
index f3eb5c2271..1a0a74de22 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java
@@ -41,6 +41,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
+import org.apache.iotdb.db.wal.WALManager;
import org.apache.iotdb.db.wal.buffer.IWALBuffer;
import org.apache.iotdb.db.wal.buffer.WALBuffer;
import org.apache.iotdb.db.wal.buffer.WALEntry;
@@ -302,8 +303,10 @@ public class WALNode implements IWALNode {
// delete files
int deletedFilesNum = 0;
for (int i = 0; i < endFileIndex; ++i) {
+ long fileSize = filesToDelete[i].length();
if (filesToDelete[i].delete()) {
deletedFilesNum++;
+ WALManager.getInstance().subtractTotalDiskUsage(fileSize);
} else {
logger.info(
"Fail to delete outdated wal file {} of wal node-{}.", filesToDelete[i], identifier);
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/WALNodeRecoverTask.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/WALNodeRecoverTask.java
index 2bd4e6e114..fe71512369 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/WALNodeRecoverTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/WALNodeRecoverTask.java
@@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@@ -110,6 +111,11 @@ public class WALNodeRecoverTask implements Runnable {
long[] indexInfo = recoverLastFile();
long lastVersionId = indexInfo[0];
long lastSearchIndex = indexInfo[1];
+ // update disk usage
+ long totalSize =
+ Arrays.stream(WALFileUtils.listAllWALFiles(logDirectory)).mapToLong(File::length).sum();
+ WALManager.getInstance().addTotalDiskUsage(totalSize);
+ // register wal node
WALManager.getInstance()
.registerWALNode(
logDirectory.getName(),