You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/06/15 09:01:35 UTC
[iotdb] branch test-snapshot-consume created (now 93259a56309)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a change to branch test-snapshot-consume
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 93259a56309 Finish
This branch includes the following new commits:
new e1d814fbc7a add snapshot consume log
new 93259a56309 Finish
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/02: add snapshot consume log
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch test-snapshot-consume
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e1d814fbc7acf990036ad2e875378d2456a0fe4c
Author: YongzaoDan <53...@qq.com>
AuthorDate: Thu Jun 15 13:40:07 2023 +0800
add snapshot consume log
---
.../iotdb/confignode/persistence/executor/ConfigPlanExecutor.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index e33c0ec0145..0aa2f9ab138 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -512,7 +512,13 @@ public class ConfigPlanExecutor {
.forEach(
x -> {
try {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info("[LoadSnapshot] Start to load snapshot for {}", x.getClass().getName());
x.processLoadSnapshot(latestSnapshotRootDir);
+ LOGGER.info(
+ "[LoadSnapshot] Load snapshot for {} cost {} ms",
+ x.getClass().getName(),
+ System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
result.set(false);
LOGGER.error("Load snapshot error: {}", e.getMessage());
[iotdb] 02/02: Finish
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch test-snapshot-consume
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 93259a56309325a3b3b000d772f8704cd5cfe8ee
Author: YongzaoDan <53...@qq.com>
AuthorDate: Thu Jun 15 17:01:23 2023 +0800
Finish
---
.../persistence/executor/ConfigPlanExecutor.java | 22 ++++++++++++++++++----
.../persistence/partition/PartitionInfo.java | 10 ++++++++--
2 files changed, 26 insertions(+), 6 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index 0aa2f9ab138..a6e7710db84 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -480,7 +480,16 @@ public class ConfigPlanExecutor {
x -> {
boolean takeSnapshotResult = true;
try {
+ long startTime = System.currentTimeMillis();
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Start to take snapshot for {} into {}",
+ x.getClass().getName(),
+ snapshotDir.getAbsolutePath());
takeSnapshotResult = x.processTakeSnapshot(snapshotDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Finish to take snapshot for {}, time consumption: {} ms",
+ x.getClass().getName(),
+ System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
LOGGER.error("Take snapshot error: {}", e.getMessage());
takeSnapshotResult = false;
@@ -493,7 +502,7 @@ public class ConfigPlanExecutor {
}
});
if (result.get()) {
- LOGGER.info("Task snapshot success, snapshotDir: {}", snapshotDir);
+ LOGGER.info("[ConfigNodeSnapshot] Task snapshot success, snapshotDir: {}", snapshotDir);
}
return result.get();
}
@@ -513,10 +522,13 @@ public class ConfigPlanExecutor {
x -> {
try {
long startTime = System.currentTimeMillis();
- LOGGER.info("[LoadSnapshot] Start to load snapshot for {}", x.getClass().getName());
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Start to load snapshot for {} from {}",
+ x.getClass().getName(),
+ latestSnapshotRootDir.getAbsolutePath());
x.processLoadSnapshot(latestSnapshotRootDir);
LOGGER.info(
- "[LoadSnapshot] Load snapshot for {} cost {} ms",
+ "[ConfigNodeSnapshot] Load snapshot for {} cost {} ms",
x.getClass().getName(),
System.currentTimeMillis() - startTime);
} catch (TException | IOException e) {
@@ -525,7 +537,9 @@ public class ConfigPlanExecutor {
}
});
if (result.get()) {
- LOGGER.info("Load snapshot success, latestSnapshotRootDir: {}", latestSnapshotRootDir);
+ LOGGER.info(
+ "[ConfigNodeSnapshot] Load snapshot success, latestSnapshotRootDir: {}",
+ latestSnapshotRootDir);
}
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
index 6771f17d75b..55f4d42c690 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
@@ -71,10 +71,11 @@ import org.apache.thrift.transport.TIOStreamTransport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedInputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -100,6 +101,9 @@ public class PartitionInfo implements SnapshotProcessor {
private static final Logger LOGGER = LoggerFactory.getLogger(PartitionInfo.class);
+ // Allocate 8MB buffer for load snapshot of PartitionInfo
+ private static final int PARTITION_TABLE_BUFFER_SIZE = 8 * 1024 * 1024;
+
/** For Cluster Partition */
// For allocating Regions
private final AtomicInteger nextRegionGroupId;
@@ -825,7 +829,9 @@ public class PartitionInfo implements SnapshotProcessor {
return;
}
- try (FileInputStream fileInputStream = new FileInputStream(snapshotFile);
+ try (BufferedInputStream fileInputStream =
+ new BufferedInputStream(
+ Files.newInputStream(snapshotFile.toPath()), PARTITION_TABLE_BUFFER_SIZE);
TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileInputStream)) {
TProtocol protocol = new TBinaryProtocol(tioStreamTransport);
// before restoring a snapshot, clear all old data