You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2020/02/14 08:37:08 UTC
[incubator-iotdb] 02/02: fix bugs of datahealthChecker
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch tsfileTools
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 60eeb83a61045850f5779c1383bb034f0b08e49c
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Fri Feb 14 16:36:41 2020 +0800
fix bugs of datahealthChecker
---
.../apache/iotdb/db/tools/DataHealthChecker.java | 68 +++++++++++++++++++---
1 file changed, 60 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java b/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java
index 7c918bf..8231f94 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.tools;
import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -46,8 +48,12 @@ import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
public class DataHealthChecker {
public static void main(String[] args) {
String dataFolder = "data/data/";//the dataFoler has two subfolders: sequence and unsequence
+ String progressFile = "progress.log";
if (args.length == 1) {
dataFolder = args[0];
+ } if (args.length == 2) {
+ dataFolder = args[0];
+ progressFile = args[1];
}
System.out.println("data folder path:" + dataFolder);
File folder = FSFactoryProducer.getFSFactory().getFile(dataFolder);
@@ -56,35 +62,81 @@ public class DataHealthChecker {
System.out.println("the input is not a data folder");
return;
}
+
File[] subFiles = folder.listFiles(File::isDirectory);
Arrays.sort(subFiles, (x, y) -> x.getName().compareTo(y.getName()));
if (subFiles.length > 2) {
System.out.println("[ERROR] there are some unknown folder under the data folder");
}
+
+ int total = 0;
+ for (File seqOrUnSeq : subFiles) {
+ for (File sgFolder : seqOrUnSeq.listFiles(File::isDirectory)) {
+ for (File partitionFolder : sgFolder.listFiles(File::isDirectory)) {
+ //check the resource file first
+ total += partitionFolder.listFiles(x -> x.getName().endsWith(".tsfile")).length;
+ }
+ }
+ }
+
+ FileWriter writer = null;
+ try {
+ writer = new FileWriter(progressFile);
+ System.out.println("The progress is recorded in " + new File(progressFile).getAbsolutePath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ return;
+ }
+
+ int current =0;
for (File seqOrUnSeq : subFiles) {
for(File sgFolder : seqOrUnSeq.listFiles(File::isDirectory)) {
for (File partitionFolder : sgFolder.listFiles(File::isDirectory)) {
//check the resource file first
for (File tsfile : partitionFolder.listFiles(x -> x.getName().endsWith(".tsfile"))) {
- //check resource
- TsFileResource tsResource = new TsFileResource(SystemFileFactory.INSTANCE.getFile(tsfile.getAbsolutePath()));
try {
- tsResource.deSerialize();
- if (tsResource.getStartTimeMap().isEmpty()) {
- System.out.print(String.format("[ERROR] Resource file %s has no devices", tsfile.getAbsolutePath()+".resource"));
+ writer.write(String.format("%d/%d\n", ++current, total));
+ writer.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ try {
+ writer.close();
+ } catch (IOException ex) {
}
- } catch (Exception e) {
- System.out.print(String.format("[ERROR] Resource file %s is broken: %s", tsfile.getAbsolutePath()+".resource", e.getMessage()));
+ return;
+ }
+ //check resource
+ if (SystemFileFactory.INSTANCE.getFile(tsfile.getAbsolutePath()+".resource").exists()) {
+ TsFileResource tsResource = new TsFileResource(
+ SystemFileFactory.INSTANCE.getFile(tsfile.getAbsolutePath()));
+ try {
+ tsResource.deSerialize();
+ if (tsResource.getStartTimeMap().isEmpty()) {
+ System.out.print(String.format("[ERROR] Resource file %s has no devices",
+ tsfile.getAbsolutePath() + ".resource"));
+ }
+ } catch (Exception e) {
+ System.out.print(String.format("[ERROR] Resource file %s is broken: %s",
+ tsfile.getAbsolutePath() + ".resource", e.getMessage()));
+ }
+ } else {
+ System.out.print(String.format("[ERROR] TsFile %s has no resource file",
+ tsfile.getAbsolutePath() ));
}
//check data
checkTsFile(tsfile);
}
for (File otherFile : partitionFolder.listFiles(x -> ! (x.getName().endsWith(".tsfile") || x.getName().endsWith(".tsfile.resource")))) {
- System.out.print(String.format("[ERROR] Bad file %s ", otherFile.getAbsolutePath()));
+ System.out.println(String.format("[ERROR] Bad file %s ", otherFile.getAbsolutePath()));
}
}
}
}
+ try {
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
private static void checkTsFile(File tsfile) {