You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2020/06/26 18:28:01 UTC
[incubator-iotdb] 01/01: [Experimental] Clear mlog when creating
snapshot
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch jira_782
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 463321f87b9098adc6e50176374943d0ccfd3525
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Sat Jun 27 02:27:29 2020 +0800
[Experimental] Clear mlog when creating snapshot
---
.../org/apache/iotdb/db/metadata/MLogWriter.java | 13 +++++++--
.../org/apache/iotdb/db/metadata/MManager.java | 34 ++++++++++------------
.../java/org/apache/iotdb/db/metadata/MTree.java | 5 ++++
3 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
index 18115d3..6b41a44 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.Map;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
@@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory;
public class MLogWriter {
private static final Logger logger = LoggerFactory.getLogger(MLogWriter.class);
+ private File logFile;
private BufferedWriter writer;
private int lineNumber;
@@ -47,7 +49,7 @@ public class MLogWriter {
}
}
- File logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName);
+ logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + logFileName);
FileWriter fileWriter = new FileWriter(logFile, true);
writer = new BufferedWriter(fileWriter);
}
@@ -160,6 +162,14 @@ public class MLogWriter {
FSFactoryProducer.getFSFactory().moveFile(tmpLogFile, logFile);
}
+ public void clear() throws IOException {
+ writer.close();
+ Files.delete(logFile.toPath());
+ FileWriter fileWriter = new FileWriter(logFile, true);
+ writer = new BufferedWriter(fileWriter);
+ lineNumber = 0;
+ }
+
private void newLine() throws IOException {
writer.newLine();
writer.flush();
@@ -172,7 +182,6 @@ public class MLogWriter {
/**
* only used for initialize a mlog file writer.
- * @param number
*/
void setLineNumber(int number) {
lineNumber = number;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index d247342..19b4118 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -228,10 +228,7 @@ public class MManager {
}
/**
- *
- * @param logFile
* @return line number of the logFile
- * @throws IOException
*/
@SuppressWarnings("squid:S3776")
private int initFromLog(File logFile) throws IOException {
@@ -248,8 +245,10 @@ public class MManager {
} else {
mtree = MTree.deserializeFrom(mtreeSnapshot);
lastSnapshotLogLineNumber = mtree.getSnapshotLineNumber();
+ logger
+ .debug("spend {} ms to deserialize mtree from snapshot", System.currentTimeMillis() - time);
+
}
- logger.debug("spend {} ms to deserialize mtree from snapshot", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
// init the metadata from the operation log
@@ -258,15 +257,6 @@ public class MManager {
try (FileReader fr = new FileReader(logFile);
BufferedReader br = new BufferedReader(fr)) {
String cmd;
- while (idx < mtree.getSnapshotLineNumber()) {
- cmd = br.readLine();
- if (cmd == null) {
- throw new IOException(String
- .format("mtree snapshot file has %d lines but the mlog.txt has only %d lines.",
- mtree.getSnapshotLineNumber(), idx));
- }
- idx++;
- }
while ((cmd = br.readLine()) != null) {
try {
operation(cmd);
@@ -276,7 +266,8 @@ public class MManager {
}
}
}
- logger.debug("spend {} ms to deserialize mtree from mlog.txt", System.currentTimeMillis() - time);
+ logger.debug("spend {} ms to deserialize mtree from mlog.txt",
+ System.currentTimeMillis() - time);
return idx;
} else if (mtreeSnapshot.exists()) {
throw new IOException("mtree snapshot file exists but mlog.txt does not exist.");
@@ -1804,15 +1795,17 @@ public class MManager {
}
if (System.currentTimeMillis() - logFile.lastModified() >= mtreeSnapshotThresholdTime
&& logWriter.getLineNumber() > lastSnapshotLogLineNumber) {
- logger.info("Start creating MTree snapshot, because {} ms elaspse.", System.currentTimeMillis() - logFile.lastModified());
+ logger.info("Start creating MTree snapshot, because {} ms elapse.",
+ System.currentTimeMillis() - logFile.lastModified());
createMTreeSnapshot();
} else if (logWriter.getLineNumber() - lastSnapshotLogLineNumber >= mtreeSnapshotInterval) {
- logger.info("Start creating MTree snapshot, because of {} new lines are added.", logWriter.getLineNumber() - lastSnapshotLogLineNumber);
+ logger.info("Start creating MTree snapshot, because of {} new lines are added.",
+ logWriter.getLineNumber() - lastSnapshotLogLineNumber);
createMTreeSnapshot();
} else {
if (logger.isDebugEnabled()) {
logger.debug(
- "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {}ms",
+ "MTree snapshot need not be created. Current mlog line number: {}, last snapshot line number: {}, time difference from last modification: {} ms",
logWriter.getLineNumber(), lastSnapshotLogLineNumber,
System.currentTimeMillis() - logFile.lastModified());
}
@@ -1823,7 +1816,8 @@ public class MManager {
lock.readLock().lock();
long time = System.currentTimeMillis();
try {
- mtree.serializeTo(mtreeSnapshotTmpPath, logWriter.getLineNumber());
+ mtree.serializeTo(mtreeSnapshotTmpPath,
+ logWriter.getLineNumber() + mtree.getSnapshotLineNumber());
lastSnapshotLogLineNumber = logWriter.getLineNumber();
File tmpFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath);
File snapshotFile = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
@@ -1831,8 +1825,10 @@ public class MManager {
Files.delete(snapshotFile.toPath());
}
if (tmpFile.renameTo(snapshotFile)) {
- logger.info("Finish creating MTree snapshot to {}, spend {}ms.", mtreeSnapshotPath, System.currentTimeMillis() - time);
+ logger.info("Finish creating MTree snapshot to {}, spend {} ms.", mtreeSnapshotPath,
+ System.currentTimeMillis() - time);
}
+ logWriter.clear();
} catch (IOException e) {
logger.warn("Failed to create MTree snapshot to {}", mtreeSnapshotPath, e);
if (SystemFileFactory.INSTANCE.getFile(mtreeSnapshotTmpPath).exists()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index b4eb3e0..6679c34 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -922,6 +922,10 @@ public class MTree implements Serializable {
}
}
+ public void setSnapshotLineNumber(int snapshotLineNumber) {
+ this.snapshotLineNumber = snapshotLineNumber;
+ }
+
public int getSnapshotLineNumber() {
return snapshotLineNumber;
}
@@ -932,6 +936,7 @@ public class MTree implements Serializable {
bw.write(String.valueOf(lineNumber));
bw.newLine();
root.serializeTo(bw);
+ setSnapshotLineNumber(lineNumber);
}
}