You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/12/07 08:32:24 UTC
[iotdb] branch accelerate_recover created (now 9fd3fec)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch accelerate_recover
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 9fd3fec accelerate recover
This branch includes the following new commits:
new 9fd3fec accelerate recover
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: accelerate recover
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch accelerate_recover
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9fd3fec49537c3b243b2aa93a8aebb33d2605c92
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue Dec 7 16:31:40 2021 +0800
accelerate recover
---
.../engine/storagegroup/StorageGroupProcessor.java | 4 ++--
.../db/engine/storagegroup/TsFileProcessor.java | 2 +-
.../db/engine/storagegroup/TsFileResource.java | 4 ++--
.../writelog/recover/TsFileRecoverPerformer.java | 28 ++++++++++++----------
4 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 41edad5..9dc2914 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -790,7 +790,7 @@ public class StorageGroupProcessor {
if (TsFileResource.getInnerCompactionCount(tsFileResource.getTsFile().getName()) > 0) {
writer =
recoverPerformer.recover(false, this::getWalDirectByteBuffer, this::releaseWalBuffer);
- if (writer.hasCrashed()) {
+ if (writer != null || writer.hasCrashed()) {
tsFileManager.addForRecover(tsFileResource, isSeq);
} else {
tsFileResource.setClosed(true);
@@ -803,7 +803,7 @@ public class StorageGroupProcessor {
recoverPerformer.recover(true, this::getWalDirectByteBuffer, this::releaseWalBuffer);
}
- if (i != tsFiles.size() - 1 || !writer.canWrite()) {
+ if (i != tsFiles.size() - 1 || writer == null || !writer.canWrite()) {
// not the last file or cannot write, just close it
tsFileResource.close();
tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index e52a43d..ee34eb4 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -1148,8 +1148,8 @@ public class TsFileProcessor {
private void endFile() throws IOException, TsFileProcessorException {
logger.info("Start to end file {}", tsFileResource);
long closeStartTime = System.currentTimeMillis();
- tsFileResource.serialize();
writer.endFile();
+ tsFileResource.serialize();
logger.info("Ended file {}", tsFileResource);
// remove this processor from Closing list in StorageGroupProcessor,
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 081a67e..d39534f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -236,8 +236,8 @@ public class TsFileResource {
/** deserialize from disk */
public void deserialize() throws IOException {
try (InputStream inputStream = fsFactory.getBufferedInputStream(file + RESOURCE_SUFFIX)) {
- readVersionNumber(inputStream);
- timeIndexType = ReadWriteIOUtils.readBytes(inputStream, 1)[0];
+ // The first byte is VERSION_NUMBER, second byte is timeIndexType.
+ timeIndexType = ReadWriteIOUtils.readBytes(inputStream, 2)[1];
timeIndex = TimeIndexLevel.valueOf(timeIndexType).getTimeIndex().deserialize(inputStream);
maxPlanIndex = ReadWriteIOUtils.readLong(inputStream);
minPlanIndex = ReadWriteIOUtils.readLong(inputStream);
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index b3797de..a11d27c 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -91,7 +91,7 @@ public class TsFileRecoverPerformer {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public RestorableTsFileIOWriter recover(
boolean needRedoWal, Supplier<ByteBuffer[]> supplier, Consumer<ByteBuffer[]> consumer)
- throws StorageGroupProcessorException {
+ throws StorageGroupProcessorException, IOException {
File file = FSFactoryProducer.getFSFactory().getFile(filePath);
if (!file.exists()) {
@@ -99,6 +99,13 @@ public class TsFileRecoverPerformer {
return null;
}
+ if (tsFileResource.resourceFileExists()) {
+ // .resource file exists, deserialize it
+ recoverResourceFromFile();
+ // return null here for skipping check TsFile
+ return null;
+ }
+
// remove corrupted part of the TsFile
RestorableTsFileIOWriter restorableTsFileIOWriter;
try {
@@ -144,18 +151,13 @@ public class TsFileRecoverPerformer {
}
private void recoverResource() throws IOException {
- if (tsFileResource.resourceFileExists()) {
- // .resource file exists, deserialize it
- recoverResourceFromFile();
- } else {
- // .resource file does not exist, read file metadata and recover tsfile resource
- try (TsFileSequenceReader reader =
- new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath())) {
- FileLoaderUtils.updateTsFileResource(reader, tsFileResource);
- }
- // write .resource file
- tsFileResource.serialize();
+ // .resource file does not exist, read file metadata and recover tsfile resource
+ try (TsFileSequenceReader reader =
+ new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath())) {
+ FileLoaderUtils.updateTsFileResource(reader, tsFileResource);
}
+ // write .resource file
+ tsFileResource.serialize();
}
private void recoverResourceFromFile() throws IOException {
@@ -295,8 +297,8 @@ public class TsFileRecoverPerformer {
tsFileResource.updatePlanIndexes(recoverMemTable.getMaxPlanIndex());
}
- restorableTsFileIOWriter.endFile();
tsFileResource.serialize();
+ restorableTsFileIOWriter.endFile();
// otherwise this file is not closed before crush, do nothing so we can continue writing
// into it