[GitHub] [incubator-iotdb] qiaojialin commented on a change in pull request #1384: [IOTDB-726] CheckPoint of MTree

[GitBox <gi...@apache.org>]

qiaojialin commented on a change in pull request #1384:
URL: https://github.com/apache/incubator-iotdb/pull/1384#discussion_r441989575



##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
##########
@@ -199,11 +203,16 @@ public synchronized void init() {
 
   private void initFromLog(File logFile) throws IOException {
     // init the metadata from the operation log
-    mtree = new MTree();
+    mtree = MTree.deserializeFrom(mtreeSnapshotPath);
     if (logFile.exists()) {
       try (FileReader fr = new FileReader(logFile);
           BufferedReader br = new BufferedReader(fr)) {
         String cmd;
+        int idx = 0;
+        while (idx <= mtree.getSnapshotLineNumber()) {

Review comment:
       should this be < ?

##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MLogWriter.java
##########
@@ -158,17 +149,23 @@ public static void upgradeMLog(String schemaDir, String logFileName) throws IOEx
         writer.write(buf.toString());
         writer.newLine();
         writer.flush();
-        
       }
     }
 
     // upgrade finished, delete old mlog file
     if (!logFile.delete()) {
       throw new IOException("Deleting " + logFile + "failed.");
     }
-    
+
     // rename tmpLogFile to mlog
     FSFactoryProducer.getFSFactory().moveFile(tmpLogFile, logFile);
   }
-  
+
+  private int newLine() throws IOException {
+    writer.newLine();
+    writer.flush();
+
+    // Every MTREE_SNAPSHOT_INTERVAL lines, create a checkpoint and save the MTree as a snapshot
+    return lineNumber++;

Review comment:
       ```suggestion
       return ++lineNumber;
   ```

##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
##########
@@ -899,6 +914,70 @@ private void findNodes(MNode node, String path, List<String> res, int targetLeve
     }
   }
 
+  public int getSnapshotLineNumber() {
+    return snapshotLineNumber;
+  }
+
+  public void serializeTo(String snapshotPath, int lineNumber) throws IOException {
+    try (BufferedWriter bw = new BufferedWriter(
+        new FileWriter(SystemFileFactory.INSTANCE.getFile(snapshotPath)))) {
+      bw.write(String.valueOf(lineNumber));
+      bw.newLine();
+      root.serializeTo(bw);
+    }
+  }
+
+  public static MTree deserializeFrom(String mtreeSnapshotPath) throws IOException {
+    File mtreeSnapshot = SystemFileFactory.INSTANCE.getFile(mtreeSnapshotPath);
+    if (!mtreeSnapshot.exists()) {
+      return new MTree();
+    }
+
+    try (BufferedReader br = new BufferedReader(new FileReader(mtreeSnapshot))) {
+      int snapshotLineNumber = Integer.parseInt(br.readLine());
+      String s;
+      Deque<MNode> nodeStack = new ArrayDeque<>();
+      MNode node = null;
+
+      while ((s = br.readLine()) != null) {
+        String[] nodeInfo = s.split(",");
+        short nodeType = Short.parseShort(nodeInfo[0]);
+        if (nodeType == MetadataConstant.STORAGE_GROUP_MNODE_TYPE) {
+          node = StorageGroupMNode.deserializeFrom(nodeInfo);
+        } else if (nodeType == MetadataConstant.MEASUREMENT_MNODE_TYPE) {
+          node = MeasurementMNode.deserializeFrom(nodeInfo);
+        } else {
+          node = new MNode(null, nodeInfo[1]);
+        }
+
+        int childrenSize = Integer.parseInt(nodeInfo[nodeInfo.length - 1]);
+        if (childrenSize == 0) {
+          nodeStack.push(node);
+        } else {
+          Map<String, MNode> childrenMap = new TreeMap<>();

Review comment:
       We use LinkedHashMap before.

##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
##########
@@ -72,27 +80,34 @@
 public class MTree implements Serializable {
 
   private static final long serialVersionUID = -4200394435237291964L;
+
   private MNode root;
+  private int snapshotLineNumber;

Review comment:
       snapshotedLineNumber is ok...

##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
##########
@@ -899,6 +914,70 @@ private void findNodes(MNode node, String path, List<String> res, int targetLeve
     }
   }
 
+  public int getSnapshotLineNumber() {
+    return snapshotLineNumber;
+  }
+
+  public void serializeTo(String snapshotPath, int lineNumber) throws IOException {
+    try (BufferedWriter bw = new BufferedWriter(
+        new FileWriter(SystemFileFactory.INSTANCE.getFile(snapshotPath)))) {
+      bw.write(String.valueOf(lineNumber));
+      bw.newLine();
+      root.serializeTo(bw);
+    }
+  }
+
+  public static MTree deserializeFrom(String mtreeSnapshotPath) throws IOException {

Review comment:
       catch all exception and return an empty MTree if meet a broken file

##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
##########
@@ -899,6 +914,70 @@ private void findNodes(MNode node, String path, List<String> res, int targetLeve
     }
   }
 
+  public int getSnapshotLineNumber() {
+    return snapshotLineNumber;
+  }
+
+  public void serializeTo(String snapshotPath, int lineNumber) throws IOException {
+    try (BufferedWriter bw = new BufferedWriter(

Review comment:
       You may need to serialize to a tmp snapshot file first and then rename it to avoid a shutdown when serializing.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org