You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2020/05/23 09:05:31 UTC
[incubator-iotdb] 01/01: Fix recover close file in each partition
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch jira-696
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 8d75f23a729c05d92eacba864e9f70de56d4ab1e
Author: HTHou <hh...@outlook.com>
AuthorDate: Sat May 23 17:04:55 2020 +0800
Fix recover close file in each partition
---
.../engine/storagegroup/StorageGroupProcessor.java | 29 ++++++++++++++--------
1 file changed, 18 insertions(+), 11 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 58a7810..61df2ba 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -269,19 +269,19 @@ public class StorageGroupProcessor {
try {
// collect candidate TsFiles from sequential and unsequential data directory
- Pair<List<TsFileResource>, List<TsFileResource>> seqTsFilesPair = getAllFiles(
+ Pair<List<List<TsFileResource>>, List<TsFileResource>> seqTsFilesPair = getAllFiles(
DirectoryManager.getInstance().getAllSequenceFileFolders());
- List<TsFileResource> tmpSeqTsFiles = seqTsFilesPair.left;
+ List<List<TsFileResource>> tmpSeqTsFilesInPartitions = seqTsFilesPair.left;
List<TsFileResource> oldSeqTsFiles = seqTsFilesPair.right;
upgradeSeqFileList.addAll(oldSeqTsFiles);
- Pair<List<TsFileResource>, List<TsFileResource>> unseqTsFilesPair = getAllFiles(
+ Pair<List<List<TsFileResource>>, List<TsFileResource>> unseqTsFilesPair = getAllFiles(
DirectoryManager.getInstance().getAllUnSequenceFileFolders());
- List<TsFileResource> tmpUnseqTsFiles = unseqTsFilesPair.left;
+ List<List<TsFileResource>> tmpUnseqTsFilesInPartitions = unseqTsFilesPair.left;
List<TsFileResource> oldUnseqTsFiles = unseqTsFilesPair.right;
upgradeUnseqFileList.addAll(oldUnseqTsFiles);
- recoverSeqFiles(tmpSeqTsFiles);
- recoverUnseqFiles(tmpUnseqTsFiles);
+ tmpSeqTsFilesInPartitions.forEach(tmpSeqTsFiles -> recoverSeqFiles(tmpSeqTsFiles));
+ tmpUnseqTsFilesInPartitions.forEach(tmpUnseqTsFiles -> recoverUnseqFiles(tmpUnseqTsFiles));
for (TsFileResource resource : sequenceFileTreeSet) {
long partitionNum = resource.getTimePartition();
@@ -392,8 +392,8 @@ public class StorageGroupProcessor {
});
}
- private Pair<List<TsFileResource>, List<TsFileResource>> getAllFiles(List<String> folders) throws IOException {
- List<File> tsFiles = new ArrayList<>();
+ private Pair<List<List<TsFileResource>>, List<TsFileResource>> getAllFiles(List<String> folders) throws IOException {
+ List<List<File>> tsFilesInPartitions = new ArrayList<>();
List<File> upgradeFiles = new ArrayList<>();
for (String baseDir : folders) {
File fileFolder = fsFactory.getFile(baseDir, storageGroupName);
@@ -458,6 +458,7 @@ public class StorageGroupProcessor {
if (!partitionFolder.isDirectory()) {
logger.warn("{} is not a directory.", partitionFolder.getAbsolutePath());
} else if (!partitionFolder.getName().equals(IoTDBConstant.UPGRADE_FOLDER_NAME)) {
+ List<File> tsFiles = new ArrayList<>();
// some TsFileResource may be being persisted when the system crashed, try recovering such
// resources
continueFailedRenames(partitionFolder, TEMP_SUFFIX);
@@ -468,14 +469,20 @@ public class StorageGroupProcessor {
Collections.addAll(tsFiles,
fsFactory.listFilesBySuffix(partitionFolder.getAbsolutePath(), TSFILE_SUFFIX));
+ tsFiles.sort(this::compareFileName);
+ tsFilesInPartitions.add(tsFiles);
}
}
}
}
- tsFiles.sort(this::compareFileName);
- List<TsFileResource> ret = new ArrayList<>();
- tsFiles.forEach(f -> ret.add(new TsFileResource(f)));
+ List<List<TsFileResource>> ret = new ArrayList<>();
+ tsFilesInPartitions.forEach(tsFiles -> {
+ List<TsFileResource> resources = new ArrayList<>();
+ tsFiles.forEach(f -> resources.add(new TsFileResource(f)));
+ ret.add(resources);
+ });
+
upgradeFiles.sort(this::compareFileName);
List<TsFileResource> upgradeRet = new ArrayList<>();
for (File f : upgradeFiles) {