You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/02/09 03:17:06 UTC
[iotdb] branch master updated: Skip broken tsfile when recovering system (#9003)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4b5f4ff5ed Skip broken tsfile when recovering system (#9003)
4b5f4ff5ed is described below
commit 4b5f4ff5ed06af9088bf87758c1998753262fa61
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Feb 9 11:17:00 2023 +0800
Skip broken tsfile when recovering system (#9003)
---
.../iotdb/db/engine/storagegroup/DataRegion.java | 115 +++++++++++----------
.../file/AbstractTsFileRecoverPerformer.java | 2 +-
2 files changed, 62 insertions(+), 55 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index d79ecbbe75..538db163ca 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -769,65 +769,72 @@ public class DataRegion implements IDataRegionForQuery {
}
private void recoverUnsealedTsFileCallBack(UnsealedTsFileRecoverPerformer recoverPerformer) {
- TsFileResource tsFileResource = recoverPerformer.getTsFileResource();
- boolean isSeq = recoverPerformer.isSequence();
- if (!recoverPerformer.canWrite()) {
- // cannot write, just close it
- for (ISyncManager syncManager :
- SyncService.getInstance().getOrCreateSyncManager(dataRegionId)) {
- syncManager.syncRealTimeTsFile(tsFileResource.getTsFile());
- }
- try {
- tsFileResource.close();
- } catch (IOException e) {
- logger.error("Fail to close TsFile {} when recovering", tsFileResource.getTsFile(), e);
- }
- updateLastFlushTime(tsFileResource, isSeq);
- tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
- TsFileMetricManager.getInstance()
- .addFile(tsFileResource.getTsFile().length(), recoverPerformer.isSequence());
- } else {
- // the last file is not closed, continue writing to it
- RestorableTsFileIOWriter writer = recoverPerformer.getWriter();
- long timePartitionId = tsFileResource.getTimePartition();
- TimePartitionManager.getInstance()
- .updateAfterOpeningTsFileProcessor(
- new DataRegionId(Integer.parseInt(dataRegionId)), timePartitionId);
- TsFileProcessor tsFileProcessor =
- new TsFileProcessor(
- dataRegionId,
- dataRegionInfo,
- tsFileResource,
- this::closeUnsealedTsFileProcessorCallBack,
- isSeq ? this::sequenceFlushCallback : this::unsequenceFlushCallback,
- isSeq,
- writer);
- if (isSeq) {
- workSequenceTsFileProcessors.put(timePartitionId, tsFileProcessor);
+ try {
+ TsFileResource tsFileResource = recoverPerformer.getTsFileResource();
+ boolean isSeq = recoverPerformer.isSequence();
+ if (!recoverPerformer.canWrite()) {
+ // cannot write, just close it
+ for (ISyncManager syncManager :
+ SyncService.getInstance().getOrCreateSyncManager(dataRegionId)) {
+ syncManager.syncRealTimeTsFile(tsFileResource.getTsFile());
+ }
+ try {
+ tsFileResource.close();
+ } catch (IOException e) {
+ logger.error("Fail to close TsFile {} when recovering", tsFileResource.getTsFile(), e);
+ }
+ updateLastFlushTime(tsFileResource, isSeq);
+ tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
+ TsFileMetricManager.getInstance()
+ .addFile(tsFileResource.getTsFile().length(), recoverPerformer.isSequence());
} else {
- workUnsequenceTsFileProcessors.put(timePartitionId, tsFileProcessor);
- }
- tsFileResource.setProcessor(tsFileProcessor);
- tsFileResource.removeResourceFile();
- tsFileProcessor.setTimeRangeId(timePartitionId);
- writer.makeMetadataVisible();
- if (enableMemControl) {
- TsFileProcessorInfo tsFileProcessorInfo = new TsFileProcessorInfo(dataRegionInfo);
- tsFileProcessor.setTsFileProcessorInfo(tsFileProcessorInfo);
- this.dataRegionInfo.initTsFileProcessorInfo(tsFileProcessor);
- // get chunkMetadata size
- long chunkMetadataSize = 0;
- for (Map<String, List<ChunkMetadata>> metaMap : writer.getMetadatasForQuery().values()) {
- for (List<ChunkMetadata> metadatas : metaMap.values()) {
- for (ChunkMetadata chunkMetadata : metadatas) {
- chunkMetadataSize += chunkMetadata.calculateRamSize();
+ // the last file is not closed, continue writing to it
+ RestorableTsFileIOWriter writer = recoverPerformer.getWriter();
+ long timePartitionId = tsFileResource.getTimePartition();
+ TimePartitionManager.getInstance()
+ .updateAfterOpeningTsFileProcessor(
+ new DataRegionId(Integer.parseInt(dataRegionId)), timePartitionId);
+ TsFileProcessor tsFileProcessor =
+ new TsFileProcessor(
+ dataRegionId,
+ dataRegionInfo,
+ tsFileResource,
+ this::closeUnsealedTsFileProcessorCallBack,
+ isSeq ? this::sequenceFlushCallback : this::unsequenceFlushCallback,
+ isSeq,
+ writer);
+ if (isSeq) {
+ workSequenceTsFileProcessors.put(timePartitionId, tsFileProcessor);
+ } else {
+ workUnsequenceTsFileProcessors.put(timePartitionId, tsFileProcessor);
+ }
+ tsFileResource.setProcessor(tsFileProcessor);
+ tsFileResource.removeResourceFile();
+ tsFileProcessor.setTimeRangeId(timePartitionId);
+ writer.makeMetadataVisible();
+ if (enableMemControl) {
+ TsFileProcessorInfo tsFileProcessorInfo = new TsFileProcessorInfo(dataRegionInfo);
+ tsFileProcessor.setTsFileProcessorInfo(tsFileProcessorInfo);
+ this.dataRegionInfo.initTsFileProcessorInfo(tsFileProcessor);
+ // get chunkMetadata size
+ long chunkMetadataSize = 0;
+ for (Map<String, List<ChunkMetadata>> metaMap : writer.getMetadatasForQuery().values()) {
+ for (List<ChunkMetadata> metadatas : metaMap.values()) {
+ for (ChunkMetadata chunkMetadata : metadatas) {
+ chunkMetadataSize += chunkMetadata.calculateRamSize();
+ }
}
}
+ tsFileProcessorInfo.addTSPMemCost(chunkMetadataSize);
}
- tsFileProcessorInfo.addTSPMemCost(chunkMetadataSize);
}
+ tsFileManager.add(tsFileResource, recoverPerformer.isSequence());
+ } catch (Throwable e) {
+ logger.error(
+ "Fail to recover unsealed TsFile {}, skip it.",
+ recoverPerformer.getTsFileAbsolutePath(),
+ e);
}
- tsFileManager.add(tsFileResource, recoverPerformer.isSequence());
}
/** recover sealed TsFile */
@@ -850,7 +857,7 @@ public class DataRegion implements IDataRegionForQuery {
sealedTsFile.close();
tsFileManager.add(sealedTsFile, isSeq);
tsFileResourceManager.registerSealedTsFileResource(sealedTsFile);
- } catch (DataRegionException | IOException e) {
+ } catch (Throwable e) {
logger.error("Fail to recover sealed TsFile {}, skip it.", sealedTsFile.getTsFilePath(), e);
} finally {
// update recovery context
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
index d8b507f643..1d9047d349 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
@@ -100,7 +100,7 @@ public abstract class AbstractTsFileRecoverPerformer implements Closeable {
private void loadResourceFile() throws IOException {
try {
tsFileResource.deserialize();
- } catch (IOException e) {
+ } catch (Throwable e) {
logger.warn(
"Cannot deserialize .resource file of {}, try to reconstruct it.",
tsFileResource.getTsFile(),