You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/22 11:37:40 UTC
[incubator-iotdb] 03/03: update directory manager
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 7d86e51924a5a0f10b7868fb00829d95b588d652
Author: qiaojialin <64...@qq.com>
AuthorDate: Sat Jun 22 19:37:35 2019 +0800
update directory manager
---
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 46 ++++++++++++----------
.../engine/modification/DeletionFileNodeTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 8 +++-
3 files changed, 33 insertions(+), 23 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index 379c4f6..de39ba8 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@ -124,45 +124,51 @@ public class FileNodeProcessorV2 {
}
private void recover() throws ProcessorException {
- List<String> tsFiles = new ArrayList<>();
- List<String> fileFolders = directoryManager.getAllTsFileFolders();
- for (String baseDir: fileFolders) {
+ List<File> tsFiles = new ArrayList<>();
+ List<String> seqFileFolders = directoryManager.getAllTsFileFolders();
+ for (String baseDir: seqFileFolders) {
File fileFolder = new File(baseDir, storageGroupName);
if (!fileFolder.exists()) {
continue;
}
for (File tsfile: fileFolder.listFiles()) {
- tsFiles.add(tsfile.getPath());
+ tsFiles.add(tsfile);
}
}
recoverSeqFiles(tsFiles);
tsFiles.clear();
- String unseqFileFolder = IoTDBDescriptor.getInstance().getConfig().getOverflowDataDir();
- File fileFolder = new File(unseqFileFolder, storageGroupName);
- if (!fileFolder.exists()) {
- return;
- }
- for (File unseqFile: fileFolder.listFiles()) {
- tsFiles.add(unseqFile.getPath());
+ List<String> unseqFileFolder = directoryManager.getAllOverflowFileFolders();
+ for (String baseDir: unseqFileFolder) {
+ File fileFolder = new File(baseDir, storageGroupName);
+ if (!fileFolder.exists()) {
+ continue;
+ }
+ for (File tsfile: fileFolder.listFiles()) {
+ tsFiles.add(tsfile);
+ }
}
recoverUnseqFiles(tsFiles);
}
- private void recoverSeqFiles(List<String> tsfiles) throws ProcessorException {
- tsfiles.sort(Comparator.comparingInt(file -> Integer.parseInt(file.split("-")[1])));
- for (String tsfile: tsfiles) {
- TsFileResourceV2 tsFileResource = new TsFileResourceV2(new File(tsfile));
+ private static int getTsFileSerialNum(File tsFile) {
+ return Integer.parseInt(tsFile.getName().split("-")[1]);
+ }
+
+ private void recoverSeqFiles(List<File> tsfiles) throws ProcessorException {
+ tsfiles.sort(Comparator.comparingInt(FileNodeProcessorV2::getTsFileSerialNum));
+ for (File tsfile: tsfiles) {
+ TsFileResourceV2 tsFileResource = new TsFileResourceV2(tsfile);
sequenceFileList.add(tsFileResource);
SeqTsFileRecoverPerformer recoverPerformer = new SeqTsFileRecoverPerformer(storageGroupName + "-", fileSchema, versionController, tsFileResource);
recoverPerformer.recover();
}
}
- private void recoverUnseqFiles(List<String> tsfiles) throws ProcessorException {
- tsfiles.sort(Comparator.comparingInt(file -> Integer.parseInt(file.split("-")[1])));
- for (String tsfile: tsfiles) {
- TsFileResourceV2 tsFileResource = new TsFileResourceV2(new File(tsfile));
+ private void recoverUnseqFiles(List<File> tsfiles) throws ProcessorException {
+ tsfiles.sort(Comparator.comparingInt(FileNodeProcessorV2::getTsFileSerialNum));
+ for (File tsfile: tsfiles) {
+ TsFileResourceV2 tsFileResource = new TsFileResourceV2(tsfile);
unSequenceFileList.add(tsFileResource);
UnSeqTsFileRecoverPerformer recoverPerformer = new UnSeqTsFileRecoverPerformer(storageGroupName + "-", fileSchema,
versionController, tsFileResource);
@@ -255,7 +261,7 @@ public class FileNodeProcessorV2 {
} else {
if (workUnSequenceTsFileProcessor == null) {
// TODO check if the disk is full, move this
- String baseDir = IoTDBDescriptor.getInstance().getConfig().getOverflowDataDir();
+ String baseDir = directoryManager.getNextFolderForOverflowFile();
new File(baseDir, storageGroupName).mkdirs();
String filePath = Paths.get(baseDir, storageGroupName, System.currentTimeMillis() + "-" + +versionController.nextVersion()).toString();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 41a5b7a..1e908ca 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -245,7 +245,7 @@ public class DeletionFileNodeTest {
new Deletion(processorName + "." + measurements[3], 105, 30),
};
- String fileNodePath = IoTDBDescriptor.getInstance().getConfig().getOverflowDataDir() + File.separator
+ String fileNodePath = DirectoryManager.getInstance().getNextFolderForOverflowFile() + File.separator
+ processorName + File.separator + "0" + File.separator;
File fileNodeDir = new File(fileNodePath);
File[] modFiles = fileNodeDir.listFiles((dir, name)
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index cde7277..45ffd51 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -103,7 +103,9 @@ public class EnvironmentUtils {
cleanDir(path);
}
// delete overflow
- cleanDir(config.getOverflowDataDir());
+ for (String path : directoryManager.getAllOverflowFileFolders()) {
+ cleanDir(path);
+ }
// delete filenode
cleanDir(config.getFileNodeDir());
// delete metadata
@@ -183,7 +185,9 @@ public class EnvironmentUtils {
createDir(path);
}
// create overflow
- createDir(config.getOverflowDataDir());
+ for (String path : directoryManager.getAllOverflowFileFolders()) {
+ cleanDir(path);
+ }
// create filenode
createDir(config.getFileNodeDir());
// create metadata