You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/07/21 03:29:13 UTC
[incubator-iotdb] branch master updated: Rm vm tmp file (#1528)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new db48ebb Rm vm tmp file (#1528)
db48ebb is described below
commit db48ebbcd15b8af71b5e7c7041c5377a63146512
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Tue Jul 21 11:29:03 2020 +0800
Rm vm tmp file (#1528)
* add vm level test and fix bug
Co-authored-by: 张凌哲 <zh...@bytedance.com>
---
.../iotdb/db/engine/flush/VmLogAnalyzer.java | 31 ++++++++++--------
.../org/apache/iotdb/db/engine/flush/VmLogger.java | 1 +
.../db/engine/storagegroup/TsFileProcessor.java | 38 ++++------------------
.../tsfile/common/constant/TsFileConstant.java | 1 -
4 files changed, 25 insertions(+), 46 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogAnalyzer.java
index 72c987f..67612b6 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogAnalyzer.java
@@ -55,19 +55,24 @@ public class VmLogAnalyzer {
String currLine;
try (BufferedReader bufferedReader = new BufferedReader(new FileReader(logFile))) {
currLine = bufferedReader.readLine();
- if (currLine != null) {
- if (currLine.equals(SOURCE_NAME)) {
- currLine = bufferedReader.readLine();
- sourceFiles.add(new File(currLine));
- } else if (currLine.equals(TARGET_NAME)) {
- currLine = bufferedReader.readLine();
- targetFile = new File(currLine);
- } else if (currLine.equals(MERGE_FINISHED)) {
- isMergeFinished = true;
- } else {
- String[] resultList = currLine.split(STR_DEVICE_OFFSET_SEPERATOR);
- deviceSet.add(resultList[0]);
- offset = Long.parseLong(resultList[1]);
+ while (currLine != null) {
+ switch (currLine) {
+ case SOURCE_NAME:
+ currLine = bufferedReader.readLine();
+ sourceFiles.add(new File(currLine));
+ break;
+ case TARGET_NAME:
+ currLine = bufferedReader.readLine();
+ targetFile = new File(currLine);
+ break;
+ case MERGE_FINISHED:
+ isMergeFinished = true;
+ break;
+ default:
+ String[] resultList = currLine.split(STR_DEVICE_OFFSET_SEPERATOR);
+ deviceSet.add(resultList[0]);
+ offset = Long.parseLong(resultList[1]);
+ break;
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogger.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogger.java
index 9419900..e075b4d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogger.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogger.java
@@ -56,6 +56,7 @@ public class VmLogger {
public void logFile(String prefix, File file) throws IOException {
logStream.write(prefix);
+ logStream.newLine();
logStream.write(file.getAbsolutePath());
logStream.newLine();
logStream.flush();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index c5b687c..ded3d8b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -1211,19 +1211,14 @@ public class TsFileProcessor {
storageGroupName, tsFileResource.getTsFile().getName(), i,
vmMergeTsFiles.get(i).size());
- // merge all vm files into a new vm file
- File tmpFile = createNewTmpFile();
- RestorableTsFileIOWriter tmpWriter = new RestorableTsFileIOWriter(tmpFile);
- VmMergeUtils.merge(tmpWriter, vmMergeWriters.get(i),
+ RestorableTsFileIOWriter newWriter = new RestorableTsFileIOWriter(newVmFile);
+ VmMergeUtils.merge(newWriter, vmMergeWriters.get(i),
storageGroupName, vmLogger, new HashSet<>(), sequence);
- tmpWriter.close();
+ newWriter.close();
vmMergeLock.writeLock().lock();
try {
deleteVmFiles(vmMergeTsFiles.get(i), vmMergeWriters.get(i));
vmLogger.logMergeFinish();
- if (tmpFile != null && !tmpFile.renameTo(newVmFile)) {
- logger.error("Failed to rename {} to {}", tmpFile, newVmFile);
- }
TsFileResource newMergedVmFile = new TsFileResource(newVmFile);
if (vmWriters.size() <= i + 1) {
vmTsFileResources.add(new CopyOnWriteArrayList<>());
@@ -1233,10 +1228,9 @@ public class TsFileProcessor {
}
vmTsFileResources.get(i + 1).add(newMergedVmFile);
vmMergeTsFiles.get(i + 1).add(newMergedVmFile);
- tmpWriter.setFile(newVmFile);
- tmpWriter.makeMetadataVisible();
- vmWriters.get(i + 1).add(tmpWriter);
- vmMergeWriters.get(i + 1).add(tmpWriter);
+ newWriter.makeMetadataVisible();
+ vmWriters.get(i + 1).add(newWriter);
+ vmMergeWriters.get(i + 1).add(newWriter);
logger.debug("{} vm file open a writer", newVmFile.getName());
} finally {
vmMergeLock.writeLock().unlock();
@@ -1260,25 +1254,5 @@ public class TsFileProcessor {
tsFileResource.getTsFile().getName(), System.currentTimeMillis() - startTimeMillis);
}
}
-
- private File createNewTmpFile() {
- vmFileCreateLock.writeLock().lock();
- try {
- TimeUnit.MILLISECONDS.sleep(1);
- File parent = writer.getFile().getParentFile();
- return FSFactoryProducer.getFSFactory().getFile(parent,
- writer.getFile().getName() + IoTDBConstant.FILE_NAME_SEPARATOR + System
- .currentTimeMillis()
- + VM_SUFFIX + IoTDBConstant.PATH_SEPARATOR
- + TMP_SUFFIX);
- } catch (InterruptedException e) {
- logger.error("{}: {}, closing task is interrupted.",
- storageGroupName, tsFileResource.getTsFile().getName(), e);
- Thread.currentThread().interrupt();
- return null;
- } finally {
- vmFileCreateLock.writeLock().unlock();
- }
- }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
index b525172..08b966a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
@@ -26,7 +26,6 @@ public class TsFileConstant {
public static final String VM_SUFFIX = ".vm";
public static final String PATH_ROOT = "root";
public static final String TMP_SUFFIX = "tmp";
- public static final String MERGED_SUFFIX = ".merged";
public static final String FLUSH_SUFFIX = ".flush";
public static final String PATH_SEPARATOR = ".";
public static final String PATH_SEPARATER_NO_REGEX = "\\.";