You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/12/14 05:56:53 UTC
[iotdb] branch rel/0.12 updated: [IOTDB-2138] [To rel/0.12] Fix data loss after IoTDB recover (#4563)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new adb5131 [IOTDB-2138] [To rel/0.12] Fix data loss after IoTDB recover (#4563)
adb5131 is described below
commit adb513100d24deed2f10291cb3e896582a8d046c
Author: liuxuxin <37...@users.noreply.github.com>
AuthorDate: Tue Dec 14 13:56:17 2021 +0800
[IOTDB-2138] [To rel/0.12] Fix data loss after IoTDB recover (#4563)
---
.../level/LevelCompactionTsFileManagement.java | 13 +++++--------
.../engine/storagegroup/StorageGroupProcessor.java | 2 ++
.../db/engine/storagegroup/TsFileResource.java | 22 ++++++++++++++++++----
3 files changed, 25 insertions(+), 12 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index aabb674..0e1c12a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -421,27 +421,24 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
CompactionFileInfo targetFileInfo = logAnalyzer.getTargetFileInfo();
String[] dataDirs = IoTDBDescriptor.getInstance().getConfig().getDataDirs();
File targetFile = null;
- boolean isSeq = logAnalyzer.isSeq();
if (targetFileInfo == null || sourceFileInfo.isEmpty()) {
return;
}
// get tsfile resource from list, as they have been recovered in StorageGroupProcessor
TsFileResource targetResource = null;
+ File targetResourceFile = null;
for (String dataDir : dataDirs) {
if ((targetFile = targetFileInfo.getFile(dataDir)).exists()) {
targetResource = new TsFileResource(targetFile);
+ targetResourceFile = new File(targetFile.getPath() + TsFileResource.RESOURCE_SUFFIX);
}
}
if (targetResource != null) {
- RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(targetFile);
- if (writer.hasCrashed()) {
- // target tsfile is not compeleted
- writer.close();
- logger.info(
- "[Compaction][Recover] target file {} is not complete, remove it", targetResource);
+ if (!targetResourceFile.exists()) {
+ // target file resource has not been generated yet
+ // delete target file if exists
targetResource.remove();
} else {
- writer.close();
// complete compaction, delete source files
logger.info(
"[Compaction][Recover] target file {} is compeleted, remove resource file",
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 7d103e7..8da66de 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -673,6 +673,8 @@ public class StorageGroupProcessor {
try {
// this tsfile is not zero level, no need to perform redo wal
if (TsFileResource.getMergeLevel(tsFileResource.getTsFile().getName()) > 0) {
+ writer =
+ recoverPerformer.recover(false, this::getWalDirectByteBuffer, this::releaseWalBuffer);
tsFileResource.setClosed(true);
tsFileManagement.add(tsFileResource, isSeq);
continue;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index ab33b23..ee9cdc3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -32,7 +32,8 @@ import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.service.UpgradeSevice;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.file.metadata.*;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -51,9 +52,21 @@ import java.io.OutputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.*;
-
-import static org.apache.iotdb.db.conf.IoTDBConstant.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_MERGECNT_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_TIME_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_VERSION_INDEX;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@SuppressWarnings("java:S1135") // ignore todos
@@ -491,6 +504,7 @@ public class TsFileResource {
public void removeResourceFile() {
try {
+ fsFactory.deleteIfExists(fsFactory.getFile(file.getPath() + RESOURCE_SUFFIX + TEMP_SUFFIX));
fsFactory.deleteIfExists(fsFactory.getFile(file.getPath() + RESOURCE_SUFFIX));
} catch (IOException e) {
logger.error("TsFileResource {} cannot be deleted: {}", file, e.getMessage());