You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/05/24 14:55:12 UTC
[iotdb] branch autoLoad0.12 created (now 78cd4c2ca7)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch autoLoad0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 78cd4c2ca7 auto load tsfile with incorrect version num
This branch includes the following new commits:
new 78cd4c2ca7 auto load tsfile with incorrect version num
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: auto load tsfile with incorrect version num
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch autoLoad0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 78cd4c2ca79fc35db6ba05af949b390b5ab61ebb
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue May 24 22:54:57 2022 +0800
auto load tsfile with incorrect version num
---
.../org/apache/iotdb/db/engine/StorageEngine.java | 15 +++++++++++++++
.../db/engine/storagegroup/StorageGroupProcessor.java | 19 ++++++++++++++++++-
.../virtualSg/VirtualStorageGroupManager.java | 4 ++++
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 4e2e81dd4c..479adfce76 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -259,6 +259,21 @@ public class StorageEngine implements IService {
}
recoveryThreadPool.shutdown();
setAllSgReady(true);
+ for (VirtualStorageGroupManager manager : processorMap.values()) {
+ for (StorageGroupProcessor processor : manager.getVirtualStorageGroupProcessor()) {
+ for (TsFileResource resource : processor.getFilesToBeLoaded()) {
+ try {
+ resource.deserialize();
+ loadNewTsFile(resource);
+ logger.warn("Auto load tsfile {}", resource);
+ } catch (Exception e) {
+ logger.error("load tsfile {} error", resource, e);
+ }
+ }
+ processor.getFilesToBeLoaded().clear();
+ }
+ }
+ logger.info("auto load tsfiles finished");
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 376125d510..972a6d564a 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -96,6 +96,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.Deque;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -271,6 +272,14 @@ public class StorageGroupProcessor {
private ScheduledExecutorService walTrimScheduleTask;
+ private Set<Long> versionsSet = new HashSet<>();
+
+ public List<TsFileResource> getFilesToBeLoaded() {
+ return filesToBeLoaded;
+ }
+
+ private List<TsFileResource> filesToBeLoaded = new ArrayList<>();
+
/**
* get the direct byte buffer from pool, each fetch contains two ByteBuffer, return null if fetch
* fails
@@ -469,6 +478,7 @@ public class StorageGroupProcessor {
for (List<TsFileResource> value : partitionTmpUnseqTsFiles.values()) {
recoverTsFiles(value, false);
}
+ versionsSet.clear();
for (TsFileResource resource : tsFileManagement.getTsFileList(true)) {
long partitionNum = resource.getTimePartition();
@@ -677,7 +687,14 @@ public class StorageGroupProcessor {
for (int i = tsFiles.size() - 1; i >= 0; i--) {
TsFileResource tsFileResource = tsFiles.get(i);
long timePartitionId = tsFileResource.getTimePartition();
-
+ long versionNum = tsFileResource.getVersion();
+ if (versionsSet.contains(versionNum)) {
+ logger.warn("Version number of {} error", tsFileResource.getTsFilePath());
+ filesToBeLoaded.add(tsFileResource);
+ continue;
+ } else {
+ versionsSet.add(versionNum);
+ }
TsFileRecoverPerformer recoverPerformer =
new TsFileRecoverPerformer(
logicalStorageGroupName
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java
index 3af47c5731..62ea9f8d05 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java
@@ -51,6 +51,10 @@ public class VirtualStorageGroupManager {
/** virtual storage group partitioner */
VirtualPartitioner partitioner = HashVirtualPartitioner.getInstance();
+ public StorageGroupProcessor[] getVirtualStorageGroupProcessor() {
+ return virtualStorageGroupProcessor;
+ }
+
/** all virtual storage group processor */
StorageGroupProcessor[] virtualStorageGroupProcessor;