You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2023/07/10 07:33:18 UTC
[iotdb] 01/01: [IOTDB-5997] Improve efficiency of ConfigNode PartitionInfo loadSnapshot
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch load_snapshot_enhancement
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 46e22ba6ecf698db0e15269b54f788c6ae16f15b
Author: YongzaoDan <33...@users.noreply.github.com>
AuthorDate: Fri Jun 16 14:35:43 2023 +0800
[IOTDB-5997] Improve efficiency of ConfigNode PartitionInfo loadSnapshot
---
.../persistence/executor/ConfigPlanExecutor.java | 24 ++++++++++++++++++++--
.../persistence/partition/PartitionInfo.java | 10 +++++++--
2 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index a502efd6afd..657cbb114db 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -436,7 +436,16 @@ public class ConfigPlanExecutor {
x -> {
boolean takeSnapshotResult = true;
try {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Start to take snapshot for {} into {}",
+ x.getClass().getName(),
+ snapshotDir.getAbsolutePath());
takeSnapshotResult = x.processTakeSnapshot(snapshotDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Finish to take snapshot for {}, time consumption: {} ms",
+ x.getClass().getName(),
+ System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
LOGGER.error("Take snapshot error: {}", e.getMessage());
takeSnapshotResult = false;
@@ -449,7 +458,7 @@ public class ConfigPlanExecutor {
}
});
if (result.get()) {
- LOGGER.info("Task snapshot success, snapshotDir: {}", snapshotDir);
+ LOGGER.info("[ConfigNodeSnapshot] Task snapshot success, snapshotDir: {}", snapshotDir);
}
return result.get();
}
@@ -468,14 +477,25 @@ public class ConfigPlanExecutor {
.forEach(
x -> {
try {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Start to load snapshot for {} from {}",
+ x.getClass().getName(),
+ latestSnapshotRootDir.getAbsolutePath());
x.processLoadSnapshot(latestSnapshotRootDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Load snapshot for {} cost {} ms",
+ x.getClass().getName(),
+ System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
result.set(false);
LOGGER.error("Load snapshot error: {}", e.getMessage());
}
});
if (result.get()) {
- LOGGER.info("Load snapshot success, latestSnapshotRootDir: {}", latestSnapshotRootDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Load snapshot success, latestSnapshotRootDir: {}",
+ latestSnapshotRootDir);
}
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
index 468d50b0070..208d0e4de1c 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
@@ -69,10 +69,11 @@ import org.apache.thrift.transport.TIOStreamTransport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedInputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -96,6 +97,9 @@ public class PartitionInfo implements SnapshotProcessor {
private static final Logger LOGGER = LoggerFactory.getLogger(PartitionInfo.class);
+ // Allocate 8MB buffer for load snapshot of PartitionInfo
+ private static final int PARTITION_TABLE_BUFFER_SIZE = 32 * 1024 * 1024;
+
/** For Cluster Partition */
// For allocating Regions
private final AtomicInteger nextRegionGroupId;
@@ -777,7 +781,9 @@ public class PartitionInfo implements SnapshotProcessor {
return;
}
- try (FileInputStream fileInputStream = new FileInputStream(snapshotFile);
+ try (BufferedInputStream fileInputStream =
+ new BufferedInputStream(
+ Files.newInputStream(snapshotFile.toPath()), PARTITION_TABLE_BUFFER_SIZE);
TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileInputStream)) {
TProtocol protocol = new TBinaryProtocol(tioStreamTransport);
// before restoring a snapshot, clear all old data