[GitHub] [iotdb] mychaow commented on a diff in pull request #6498: [IOTDB-3656] Loader Tool for TsFile

[GitBox <gi...@apache.org>]

mychaow commented on code in PR #6498:
URL: https://github.com/apache/iotdb/pull/6498#discussion_r909691550


##########
load-tsfile-tool/src/main/java/org/apache/iotdb/RewriteFileTool.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * This tool reads tsFiles and rewrites it chunk by chunk. It constructs tablet and invokes
+ * insertTablet() for every page in chunk, in case chunk is too large. `Move` command is used to
+ * unload files in iotdb, and mods files are moved manually.
+ */
+public class RewriteFileTool {
+  // backup data dir path
+  private static String backUpDirPath;
+  // validation file path
+  private static String validationFilePath;
+  // tsfile list path
+  private static String tsfileListPath;
+  // output file path
+  private static String outputLogFilePath;
+
+  private static final String HostIP = "localhost";
+  private static final String rpcPort = "6667";
+  private static String user = "root";
+  private static String password = "root";
+
+  private static final FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+  private static final long MAX_TABLET_SIZE = 1024 * 1024;
+
+  private static PrintWriter writer;
+
+  /**
+   * -b=[path of backUp directory] -vf=[path of validation file]/-f=[path of tsfile list] -o=[path
+   * of output log] -u=[username, default="root"] -pw=[password, default="root"]
+   */
+  public static void main(String[] args) throws IOException {
+    if (!checkArgs(args)) {
+      System.exit(1);
+    }
+    writer = new PrintWriter(new FileWriter(outputLogFilePath));
+    try {
+      if (validationFilePath != null) {
+        readValidationFile(validationFilePath);
+      }
+      if (tsfileListPath != null) {
+        readTsFileList(tsfileListPath);
+      }
+    } catch (Exception e) {
+      printBoth(e.getMessage());
+      e.printStackTrace();
+    } finally {
+      writer.close();
+    }
+  }
+
+  public static void readValidationFile(String validationFilePath)
+      throws IOException, IoTDBConnectionException {
+    Session session = new Session(HostIP, rpcPort, user, password);
+    session.open(false);
+
+    BufferedReader bufferedReader = new BufferedReader(new FileReader(validationFilePath));
+    String line;
+    while ((line = bufferedReader.readLine()) != null) {
+      if (!line.startsWith("-- Find the bad file ")) {
+        continue;
+      }
+      String badFilePath = line.replace("-- Find the bad file ", "");
+      unloadAndReWriteWrongTsFile(badFilePath, session);
+    }
+    bufferedReader.close();
+    session.close();
+    printBoth("Finish rewriting all bad files.");
+  }
+
+  public static void readTsFileList(String tsfileListPath)
+      throws IoTDBConnectionException, IOException {
+    Session session = new Session(HostIP, rpcPort, user, password);
+    session.open(false);
+
+    BufferedReader bufferedReader = new BufferedReader(new FileReader(tsfileListPath));
+    String badFilePath;
+    while ((badFilePath = bufferedReader.readLine()) != null) {
+      unloadAndReWriteWrongTsFile(badFilePath, session);
+    }
+    bufferedReader.close();
+    session.close();
+    printBoth("Finish rewriting all bad files.");
+  }
+
+  public static void unloadAndReWriteWrongTsFile(String filename, Session session) {
+    try {
+      String[] dirs = filename.split("/");
+      String targetFilePath = backUpDirPath + File.separator + dirs[dirs.length - 1];
+      File targetFile = new File(targetFilePath);
+      // move mods file
+      File modsFile = new File(filename + ModificationFile.FILE_SUFFIX);
+      if (modsFile.exists()) {
+        fsFactory.moveFile(modsFile, new File(targetFilePath + ModificationFile.FILE_SUFFIX));
+      }
+      if (targetFile.exists()) {
+        printBoth(String.format("%s is already in the backup dir. Don't need to move.", filename));
+      } else {
+        printBoth(String.format("Start moving %s to backup dir.", filename));
+        session.executeNonQueryStatement(String.format("move '%s' '%s'", filename, backUpDirPath));
+      }
+      printBoth(String.format("Finish unloading %s.", filename));
+
+      // try to rewriteFile
+      printBoth(String.format("Start rewriting %s to iotdb.", filename));
+      if (targetFile.exists()) {
+        rewriteWrongTsFile(targetFilePath, session);
+        targetFile.renameTo(new File(targetFilePath + "." + "finish"));
+      } else {
+        printBoth("---- Meet error in rewriting, " + targetFilePath + " does not exist.");
+      }
+    } catch (IoTDBConnectionException | StatementExecutionException e) {
+      e.printStackTrace();
+      printBoth("---- Meet error in unloading " + filename + ", " + e.getMessage());
+    }
+  }
+
+  public static void rewriteWrongTsFile(String filename, Session session) {
+    // read mods file
+    List<Modification> modifications = null;
+    if (FSFactoryProducer.getFSFactory()
+        .getFile(filename + ModificationFile.FILE_SUFFIX)
+        .exists()) {
+      modifications =
+          (List<Modification>)
+              new ModificationFile(filename + ModificationFile.FILE_SUFFIX).getModifications();
+    }
+
+    try (TsFileSequenceReader reader = new TsFileSequenceReader(filename)) {

Review Comment:
   the new reader could parse the old tsfile? like tsfile in 0.12 or 0.13?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org