You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/05/27 02:07:50 UTC
[iotdb] branch rewriteTool updated (fb00126fef -> a89bb5d7b0)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from fb00126fef sort time partition dir (#6041)
new ae74a2f221 handle one fiel once
new fe74f3e0ce modify comments
new a89bb5d7b0 Merge branch 'rewriteTool' of github.com:apache/iotdb into rewriteTool
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/iotdb/RewriteBadFileTool.java | 48 +++++++---------------
1 file changed, 14 insertions(+), 34 deletions(-)
[iotdb] 03/03: Merge branch 'rewriteTool' of github.com:apache/iotdb into rewriteTool
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a89bb5d7b0daf749032141c762a1c502cf0cfdec
Merge: fe74f3e0ce fb00126fef
Author: Alima777 <wx...@gmail.com>
AuthorDate: Fri May 27 10:07:30 2022 +0800
Merge branch 'rewriteTool' of github.com:apache/iotdb into rewriteTool
rewriteBadFile/pom.xml | 4 ++--
.../src/main/java/org/apache/iotdb/TsFileValidationTool.java | 10 +++++++---
2 files changed, 9 insertions(+), 5 deletions(-)
[iotdb] 01/03: handle one fiel once
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae74a2f22147b2ccd79df8354a615f4a38ed2ad2
Author: Alima777 <wx...@gmail.com>
AuthorDate: Fri May 27 10:05:57 2022 +0800
handle one fiel once
---
.../java/org/apache/iotdb/RewriteBadFileTool.java | 48 +++++++---------------
1 file changed, 14 insertions(+), 34 deletions(-)
diff --git a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
index a33ccbfafd..e1a94107be 100644
--- a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
+++ b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
@@ -65,8 +65,6 @@ public class RewriteBadFileTool {
private static String validationFilePath = "TsFile_validation_view.txt";
// output file path
private static String outputLogFilePath = "TsFile_rewrite_view.txt";
- // whether moving the data files
- private static boolean moveFile = true;
private static final String HostIP = "localhost";
private static final String rpcPort = "6667";
@@ -87,11 +85,7 @@ public class RewriteBadFileTool {
}
pw = new PrintWriter(new FileWriter(outputLogFilePath));
try {
- if (moveFile) {
- moveBadFileToBackUp();
- } else {
- rewriteAllBadFiles();
- }
+ moveAndRewriteBadFile();
} catch (IoTDBConnectionException | IOException e) {
e.printStackTrace();
} finally {
@@ -99,8 +93,10 @@ public class RewriteBadFileTool {
}
}
- public static void moveBadFileToBackUp() throws IOException {
- printBoth("Start moving bad files to backup dir.");
+ public static void moveAndRewriteBadFile() throws IOException, IoTDBConnectionException {
+ Session session = new Session(HostIP, rpcPort, user, password);
+ session.open(false);
+
BufferedReader bufferedReader = new BufferedReader(new FileReader(validationFilePath));
String line;
while ((line = bufferedReader.readLine()) != null) {
@@ -108,6 +104,7 @@ public class RewriteBadFileTool {
continue;
}
String badFilePath = line.replace("-- Find the bad file ", "");
+ printBoth(String.format("Start moving %s to backup dir.", badFilePath));
String targetFilePath =
backUpDirPath + File.separator + "sequence" + badFilePath.split("sequence")[1];
File targetFile = new File(targetFilePath);
@@ -125,26 +122,10 @@ public class RewriteBadFileTool {
if (modsFile.exists()) {
fsFactory.moveFile(modsFile, new File(targetFilePath + ModificationFile.FILE_SUFFIX));
}
- }
- bufferedReader.close();
- printBoth("Finish moving all bad files to backup dir.");
- }
-
- private static void rewriteAllBadFiles() throws IOException, IoTDBConnectionException {
- printBoth("Start rewriting bad files to iotdb.");
- Session session = new Session(HostIP, rpcPort, user, password);
- session.open(false);
- BufferedReader bufferedReader = new BufferedReader(new FileReader(validationFilePath));
- String line;
- while ((line = bufferedReader.readLine()) != null) {
- if (!line.startsWith("-- Find the bad file ")) {
- continue;
- }
- String badFilePath = line.replace("-- Find the bad file ", "");
- String targetFilePath =
- backUpDirPath + File.separator + "sequence" + badFilePath.split("sequence")[1];
+ printBoth("Finish moving.");
+ // rewriteFile
try {
- File targetFile = new File(targetFilePath);
+ printBoth(String.format("Start rewriting %s to iotdb.", badFilePath));
if (targetFile.exists()) {
rewriteWrongTsFile(targetFilePath, session);
targetFile.renameTo(new File(targetFilePath + "." + "finish"));
@@ -156,8 +137,9 @@ public class RewriteBadFileTool {
printBoth("---- Meet error in rewriting " + targetFilePath + ", " + e.getMessage());
}
}
+ bufferedReader.close();
session.close();
- printBoth("Finish rewriting all bad files to iotdb.");
+ printBoth("Finish moving all bad files to backup dir.");
}
public static void rewriteWrongTsFile(String filename, Session session)
@@ -256,9 +238,9 @@ public class RewriteBadFileTool {
}
private static boolean checkArgs(String[] args) {
- if (args.length != 4) {
+ if (args.length != 3) {
System.out.println(
- "Param incorrect, -m=[need moving data file or not] -b=[path of backUp directory] -v=[path of validation file] -o=[path of output file].");
+ "Param incorrect, -b=[path of backUp directory] -v=[path of validation file] -o=[path of output file].");
return false;
}
for (String arg : args) {
@@ -268,11 +250,9 @@ public class RewriteBadFileTool {
validationFilePath = arg.split("=")[1];
} else if (arg.startsWith("-o")) {
outputLogFilePath = arg.split("=")[1];
- } else if (arg.startsWith("-m")) {
- moveFile = Boolean.parseBoolean(arg.split("=")[1]);
} else {
System.out.println(
- "Param incorrect, -m=[need moving data file or not] -b=[path of backUp directory] -v=[path of validation file] -o=[path of output file].");
+ "Param incorrect, -b=[path of backUp directory] -v=[path of validation file] -o=[path of output file].");
return false;
}
}
[iotdb] 02/03: modify comments
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fe74f3e0ce8fbc66203a57c7409996f9b9cfedd5
Author: Alima777 <wx...@gmail.com>
AuthorDate: Fri May 27 10:07:07 2022 +0800
modify comments
---
rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
index e1a94107be..13d157e967 100644
--- a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
+++ b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
@@ -139,7 +139,7 @@ public class RewriteBadFileTool {
}
bufferedReader.close();
session.close();
- printBoth("Finish moving all bad files to backup dir.");
+ printBoth("Finish rewriting all bad files.");
}
public static void rewriteWrongTsFile(String filename, Session session)