You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/06/26 18:28:00 UTC
[incubator-iotdb] branch jira_782 created (now 463321f)
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a change to branch jira_782
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 463321f [Experimental] Clear mlog when creating snapshot
This branch includes the following new commits:
new 463321f [Experimental] Clear mlog when creating snapshot
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: [Experimental] Clear mlog when creating
snapshot
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_782
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 463321f87b9098adc6e50176374943d0ccfd3525
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Sat Jun 27 02:27:29 2020 +0800
[Experimental] Clear mlog when creating snapshot
---
.../org/apache/iotdb/db/metadata/MLogWriter.java | 13 +++++++--
.../org/apache/iotdb/db/metadata/MManager.java | 34 ++++++++++------------
.../java/org/apache/iotdb/db/metadata/MTree.java | 5 ++++
3 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
index 18115d3..6b41a44 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.Map;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
@@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory;
public class MLogWriter {
private static final Logger logger = LoggerFactory.getLogger(MLogWriter.class);
+ private File logFile;
private BufferedWriter writer;
private int lineNumber;
@@ -47,7 +49,7 @@ public class MLogWriter {
}
}
- File logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName);
+ logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName);
FileWriter fileWriter = new FileWriter(logFile, true);
writer = new BufferedWriter(fileWriter);
}
@@ -160,6 +162,14 @@ public class MLogWriter {
FSFactoryProducer.getFSFactory().moveFile(tmpLogFile, logFile);
}
+ public void clear() throws IOException {
+ writer.close();
+ Files.delete(logFile.toPath());
+ FileWriter fileWriter = new FileWriter(logFile, true);
+ writer = new BufferedWriter(fileWriter);
+ lineNumber = 0;
+ }
+
private void newLine() throws IOException {
writer.newLine();
writer.flush();
@@ -172,7 +182,6 @@ public class MLogWriter {
/**
* only used for initialize a mlog file writer.
- * @param number
*/
void setLineNumber(int number) {
lineNumber = number;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index d247342..19b4118 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -228,10 +228,7 @@ public class MManager {
}
/**
- *
- * @param logFile
* @return line number of the logFile
- * @throws IOException
*/
@SuppressWarnings("squid:S3776")
private int initFromLog(File logFile) throws IOException {
@@ -248,8 +245,10 @@ public class MManager {
} else {
mtree = MTree.deserializeFrom(mtreeSnapshot);
lastSnapshotLogLineNumber = mtree.getSnapshotLineNumber();
+ logger
+ .debug("spend {} ms to deserialize mtree from snapshot", System.currentTimeMillis() - time);
+
}
- logger.debug("spend {} ms to deserialize mtree from snapshot", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
// init the metadata from the operation log
@@ -258,15 +257,6 @@ public class MManager {
try (FileReader fr = new FileReader(logFile);
BufferedReader br = new BufferedReader(fr)) {
String cmd;
- while (idx < mtree.getSnapshotLineNumber()) {
- cmd = br.readLine();
- if (cmd == null) {
- throw new IOException(String
- .format("mtree snapshot file has %d lines but the mlog.txt has only %d lines.",
- mtree.getSnapshotLineNumber(), idx));
- }
- idx++;
- }
while ((cmd = br.readLine()) != null) {
try {
operation(cmd);
@@ -276,7 +266,8 @@ public class MManager {
}
}
}
- logger.debug("spend {} ms to deserialize mtree from mlog.txt", System.currentTimeMillis() - time);
+ logger.debug("spend {} ms to deserialize mtree from mlog.txt",
+ System.currentTimeMillis() - time);
return idx;
} else if (mtreeSnapshot.exists()) {
throw new IOException("mtree snapshot file exists but mlog.txt does not exist.");
@@ -1804,15 +1795,17 @@ public class MManager {
}
if (System.currentTimeMillis() - logFile.lastModified() >= mtreeSnapshotThresholdTime
&& logWriter.getLineNumber() > lastSnapshotLogLineNumber) {
- logger.info("Start creating MTree snapshot, because {} ms elaspse.", System.currentTimeMillis() - logFile.lastModified());
+ logger.info("Start creating MTree snapshot, because {} ms elapse.",
+ System.currentTimeMillis() - logFile.lastModified());
createMTreeSnapshot();
} else if (logWriter.getLineNumber() - lastSnapshotLogLineNumber >= mtreeSnapshotInterval) {
- logger.info("Start creating MTree snapshot, because of {} new lines are added.", logWriter.getLineNumber() - lastSnapshotLogLineNumber);
+ logger.info("Start creating MTree snapshot, because of {} new lines are added.",
+ logWriter.getLineNumber() - lastSnapshotLogLineNumber);
createMTreeSnapshot();
} else {
if (logger.isDebugEnabled()) {
logger.debug(
- "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {}ms",
+ "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {} ms",
logWriter.getLineNumber(), lastSnapshotLogLineNumber,
System.currentTimeMillis() - logFile.lastModified());
}
@@ -1823,7 +1816,8 @@ public class MManager {
lock.readLock().lock();
long time = System.currentTimeMillis();
try {
- mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber());
+ mtree.serializeTo(mtreeSnapshotTmpPath,
+ logWriter.getLineNumber() + mtree.getSnapshotLineNumber());
lastSnapshotLogLineNumber = logWriter.getLineNumber();
File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
@@ -1831,8 +1825,10 @@ public class MManager {
Files.delete(snapshotFile.toPath());
}
if (tmpFile.renameTo(snapshotFile)) {
- logger.info("Finish creating MTree snapshot to {}, spend {}ms.", mtreeSnapshotPath, System.currentTimeMillis() - time);
+ logger.info("Finish creating MTree snapshot to {}, spend {} ms.", mtreeSnapshotPath,
+ System.currentTimeMillis() - time);
}
+ logWriter.clear();
} catch (IOException e) {
logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e);
if (SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath).exists()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index b4eb3e0..6679c34 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -922,6 +922,10 @@ public class MTree implements Serializable {
}
}
+ public void setSnapshotLineNumber(int snapshotLineNumber) {
+ this.snapshotLineNumber = snapshotLineNumber;
+ }
+
public int getSnapshotLineNumber() {
return snapshotLineNumber;
}
@@ -932,6 +936,7 @@ public class MTree implements Serializable {
bw.write(String.valueOf(lineNumber));
bw.newLine();
root.serializeTo(bw);
+ setSnapshotLineNumber(lineNumber);
}
}