You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/06/25 07:49:57 UTC
[iotdb] branch addRewriteTool0.13 created (now c5ba4314fe)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch addRewriteTool0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at c5ba4314fe rename move to unload
This branch includes the following new commits:
new df0b177076 [To rel/0.13] Validation and Rewrite file tool (#6119)
new e6c1f2e214 update RewriteFileTool.java (#6166)
new c5ba4314fe rename move to unload
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/03: update RewriteFileTool.java (#6166)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch addRewriteTool0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e6c1f2e21479b8996dcbcf60404d12e41ca8d40d
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Mon Jun 6 15:57:25 2022 +0800
update RewriteFileTool.java (#6166)
---
.../java/org/apache/iotdb/RewriteFileTool.java | 74 +++++++++++++++++-----
1 file changed, 57 insertions(+), 17 deletions(-)
diff --git a/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
index 17b5af7cfb..c796548479 100644
--- a/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
+++ b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.session.Session;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
@@ -43,6 +44,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.write.record.Tablet;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -74,22 +76,23 @@ public class RewriteFileTool {
private static final String HostIP = "localhost";
private static final String rpcPort = "6667";
- private static final String user = "root";
- private static final String password = "root";
+ private static String user = "root";
+ private static String password = "root";
private static final FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+ private static final long MAX_TABLET_SIZE = 1024 * 1024;
- private static PrintWriter pw;
+ private static PrintWriter writer;
/**
* -b=[path of backUp directory] -vf=[path of validation file]/-f=[path of tsfile list] -o=[path
- * of output log]
+ * of output log] -u=[username, default="root"] -pw=[password, default="root"]
*/
public static void main(String[] args) throws IOException {
if (!checkArgs(args)) {
System.exit(1);
}
- pw = new PrintWriter(new FileWriter(outputLogFilePath));
+ writer = new PrintWriter(new FileWriter(outputLogFilePath));
try {
if (validationFilePath != null) {
readValidationFile(validationFilePath);
@@ -101,7 +104,7 @@ public class RewriteFileTool {
printBoth(e.getMessage());
e.printStackTrace();
} finally {
- pw.close();
+ writer.close();
}
}
@@ -140,9 +143,7 @@ public class RewriteFileTool {
}
public static void unloadAndReWriteWrongTsFile(String filename, Session session) {
- printBoth(String.format("Start moving %s to backup dir.", filename));
try {
- session.executeNonQueryStatement(String.format("move '%s' '%s'", filename, backUpDirPath));
String[] dirs = filename.split("/");
String targetFilePath = backUpDirPath + File.separator + dirs[dirs.length - 1];
File targetFile = new File(targetFilePath);
@@ -151,9 +152,15 @@ public class RewriteFileTool {
if (modsFile.exists()) {
fsFactory.moveFile(modsFile, new File(targetFilePath + ModificationFile.FILE_SUFFIX));
}
+ if (targetFile.exists()) {
+ printBoth(String.format("%s is already in the backup dir. Don't need to move.", filename));
+ } else {
+ printBoth(String.format("Start moving %s to backup dir.", filename));
+ session.executeNonQueryStatement(String.format("move '%s' '%s'", filename, backUpDirPath));
+ }
printBoth(String.format("Finish unloading %s.", filename));
- // rewriteFile
+ // try to rewriteFile
printBoth(String.format("Start rewriting %s to iotdb.", filename));
if (targetFile.exists()) {
rewriteWrongTsFile(targetFilePath, session);
@@ -230,17 +237,46 @@ public class RewriteFileTool {
} else {
maxRow = batchData.length();
}
+
Tablet tablet =
new Tablet(curDevice, Collections.singletonList(measurementSchema), maxRow);
- int rowIndex = 0;
+ long curTabletSize = 0;
while (batchData.hasCurrent()) {
- tablet.addTimestamp(rowIndex, batchData.currentTime());
- tablet.addValue(measurement, rowIndex, batchData.currentValue());
+ tablet.addTimestamp(tablet.rowSize, batchData.currentTime());
+ tablet.addValue(measurement, tablet.rowSize, batchData.currentValue());
+ tablet.rowSize++;
+ // calculate curTabletSize based on timestamp and value
+ curTabletSize += 8;
+ switch (header.getDataType()) {
+ case BOOLEAN:
+ curTabletSize += 1;
+ break;
+ case INT32:
+ case FLOAT:
+ curTabletSize += 4;
+ break;
+ case INT64:
+ case DOUBLE:
+ curTabletSize += 8;
+ break;
+ case TEXT:
+ curTabletSize += 4 + ((Binary) batchData.currentValue()).getLength();
+ break;
+ default:
+ throw new UnSupportedDataTypeException(
+ String.format("Data type %s is not supported.", header.getDataType()));
+ }
+ // if curTabletSize is over the threshold
+ if (curTabletSize >= MAX_TABLET_SIZE) {
+ session.insertTablet(tablet);
+ curTabletSize = 0;
+ tablet.reset();
+ }
batchData.next();
- rowIndex++;
}
- tablet.rowSize = rowIndex;
- session.insertTablet(tablet);
+ if (tablet.rowSize > 0) {
+ session.insertTablet(tablet);
+ }
dataSize -= pageHeader.getSerializedPageSize();
}
break;
@@ -269,7 +305,7 @@ public class RewriteFileTool {
private static boolean checkArgs(String[] args) {
String paramConfig =
- "-b=[path of backUp directory] -vf=[path of validation file]/-f=[path of tsfile list] -o=[path of output log]";
+ "-b=[path of backUp directory] -vf=[path of validation file]/-f=[path of tsfile list] -o=[path of output log] -u=[username, default=\"root\"] -pw=[password, default=\"root\"]";
for (String arg : args) {
if (arg.startsWith("-b")) {
backUpDirPath = arg.substring(arg.indexOf('=') + 1);
@@ -279,6 +315,10 @@ public class RewriteFileTool {
tsfileListPath = arg.substring(arg.indexOf('=') + 1);
} else if (arg.startsWith("-o")) {
outputLogFilePath = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-u")) {
+ user = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-pw")) {
+ password = arg.substring(arg.indexOf('=') + 1);
} else {
System.out.println("Param incorrect!" + paramConfig);
return false;
@@ -295,7 +335,7 @@ public class RewriteFileTool {
private static void printBoth(String msg) {
System.out.println(msg);
- pw.println(msg);
+ writer.println(msg);
}
private static List<TimeRange> getOldSortedDeleteIntervals(
[iotdb] 03/03: rename move to unload
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch addRewriteTool0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c5ba4314fecd97dc62239fc7220df5092cc14939
Author: Alima777 <wx...@gmail.com>
AuthorDate: Sat Jun 25 15:49:42 2022 +0800
rename move to unload
---
rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
index c796548479..f37f3fc9e8 100644
--- a/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
+++ b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
@@ -61,7 +61,7 @@ import java.util.List;
/**
* This tool reads tsFiles and rewrites it chunk by chunk. It constructs tablet and invokes
- * insertTablet() for every page in chunk, in case chunk is too large. `Move` command is used to
+ * insertTablet() for every page in chunk, in case chunk is too large. `Unload` command is used to
* unload files in iotdb, and mods files are moved manually.
*/
public class RewriteFileTool {
@@ -156,7 +156,8 @@ public class RewriteFileTool {
printBoth(String.format("%s is already in the backup dir. Don't need to move.", filename));
} else {
printBoth(String.format("Start moving %s to backup dir.", filename));
- session.executeNonQueryStatement(String.format("move '%s' '%s'", filename, backUpDirPath));
+ session.executeNonQueryStatement(
+ String.format("unload '%s' '%s'", filename, backUpDirPath));
}
printBoth(String.format("Finish unloading %s.", filename));
[iotdb] 01/03: [To rel/0.13] Validation and Rewrite file tool (#6119)
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch addRewriteTool0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit df0b177076152e968aeabe773eb210f3217f48c2
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Sun Jun 5 12:50:44 2022 +0800
[To rel/0.13] Validation and Rewrite file tool (#6119)
---
.gitignore | 4 +-
pom.xml | 179 +++------
rewriteFileTool/pom.xml | 82 +++++
.../assembly/resources/sbin/rewrite-file-tool.sh | 48 +++
.../src/assembly/resources/sbin/validate-tsfile.sh | 48 +++
rewriteFileTool/src/assembly/rewriteFileTool.xml | 40 ++
.../java/org/apache/iotdb/RewriteFileTool.java | 326 +++++++++++++++++
.../org/apache/iotdb/TsFileValidationTool.java | 403 +++++++++++++++++++++
8 files changed, 1006 insertions(+), 124 deletions(-)
diff --git a/.gitignore b/.gitignore
index 6b90b2f1f5..787730f204 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
**/logs/*
**/lib/**
**/data/**
+**/raft/**
# Python runtime file
**/__pycache__/**
@@ -40,7 +41,6 @@ tsfile-jdbc/src/main/resources/output/queryRes.csv
*.gz
*.tar.gz
*.tar
-*.tokens
#src/test/resources/logback.xml
### CSV ###
@@ -53,7 +53,6 @@ grafana/.mvn/
grafana/logs/
*.log
-
### STS ###
.apt_generated
.classpath
@@ -114,3 +113,4 @@ node3/
antlr/gen/
antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/gen/
antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.tokens
+*.tokens
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index ab27b70744..6ca34dd7e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
</parent>
<groupId>org.apache.iotdb</groupId>
<artifactId>iotdb-parent</artifactId>
- <version>0.13.1-SNAPSHOT</version>
+ <version>0.12.6-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache IoTDB Project Parent POM</name>
<description>This is the top level project that builds, packages the tsfile, iotdb engine, jdbc, and integration libs.</description>
@@ -42,7 +42,7 @@
<connection>scm:git:ssh://git@github.com/apache/iotdb.git</connection>
<developerConnection>scm:git:ssh://git@github.com/apache/iotdb.git</developerConnection>
<url>ssh://git@github.com:apache/iotdb.git</url>
- <tag>rel/0.10</tag>
+ <tag>v0.12.3</tag>
</scm>
<!-- Only configure the site distribution as the rest is handled by the apache parent -->
<distributionManagement>
@@ -84,16 +84,13 @@
<module>thrift</module>
<module>thrift-cluster</module>
<module>thrift-sync</module>
- <module>thrift-influxdb</module>
<module>service-rpc</module>
<module>jdbc</module>
<module>session</module>
<module>cli</module>
- <module>openapi</module>
<module>server</module>
<module>example</module>
- <module>grafana-plugin</module>
- <module>grafana-connector</module>
+ <module>grafana</module>
<module>spark-tsfile</module>
<module>hadoop</module>
<module>spark-iotdb-connector</module>
@@ -107,47 +104,44 @@
<module>client-py</module>
<module>compile-tools</module>
<module>client-cpp</module>
- <module>metrics</module>
- <module>integration</module>
- <!-- <module>library-udf</module>-->
+ <module>rewriteFileTool</module>
</modules>
<!-- Properties Management -->
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
- <maven.assembly.version>3.1.0</maven.assembly.version>
+ <maven.assembly.version>3.1.1</maven.assembly.version>
<scala.library.version>2.11</scala.library.version>
<scala.version>2.11.12</scala.version>
- <hadoop2.version>2.10.1</hadoop2.version>
+ <hadoop2.version>2.7.3</hadoop2.version>
<hive2.version>2.3.6</hive2.version>
<junit.version>4.13.2</junit.version>
<slf4j.version>1.7.12</slf4j.version>
<logback.version>1.2.10</logback.version>
<joda.version>2.9.9</joda.version>
<spark.version>2.4.5</spark.version>
- <flink.version>1.14.0</flink.version>
- <common.io.version>2.11.0</common.io.version>
+ <flink.version>1.11.1</flink.version>
+ <common.io.version>2.5</common.io.version>
<commons.collections4>4.4</commons.collections4>
<!-- keep consistent with client-cpp/tools/thrift/pom.xml-->
<thrift.version>0.14.1</thrift.version>
<airline.version>0.9</airline.version>
- <jackson.version>2.10.5</jackson.version>
+ <jackson.version>2.11.0</jackson.version>
<antlr4.version>4.8-1</antlr4.version>
<common.cli.version>1.3.1</common.cli.version>
- <common.codec.version>1.13</common.codec.version>
+ <common.codec.version>1.15</common.codec.version>
<common.collections.version>3.2.2</common.collections.version>
- <common.lang3.version>3.8.1</common.lang3.version>
+ <common.lang3.version>3.12.0</common.lang3.version>
<common.logging.version>1.1.3</common.logging.version>
- <common.pool2.version>2.11.1</common.pool2.version>
<org.slf4j.version>1.7.32</org.slf4j.version>
<guava.version>24.1.1</guava.version>
- <jline.version>3.21.0</jline.version>
+ <jline.version>2.14.6</jline.version>
<jetty.version>9.4.35.v20201120</jetty.version>
- <metrics.version>4.2.7</metrics.version>
- <javax.xml.bind.version>2.4.0-b180830.0359</javax.xml.bind.version>
+ <metrics.version>3.2.6</metrics.version>
+ <javax.xml.bind.version>2.4.0-b180725.0427</javax.xml.bind.version>
<felix.version>5.1.1</felix.version>
<snappy.version>1.1.8.4</snappy.version>
- <netty.version>4.1.53.Final</netty.version>
+ <netty.version>4.1.73.Final</netty.version>
<!-- URL of the ASF SonarQube server -->
<sonar.host.url>https://sonarcloud.io</sonar.host.url>
<sonar.organization>apache</sonar.organization>
@@ -157,7 +151,7 @@
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-merged-reports/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
<sonar.junit.reportPaths>target/surefire-reports,target/failsafe-reports</sonar.junit.reportPaths>
<!-- By default, the argLine is empty-->
- <gson.version>2.8.8</gson.version>
+ <gson.version>2.8.6</gson.version>
<argLine/>
<!-- whether enable compiling the cpp client-->
<client-cpp>false</client-cpp>
@@ -166,10 +160,6 @@
<spotless.version>2.4.2</spotless.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.13</httpcore.version>
- <!-- for REST service -->
- <swagger.core.version>1.5.18</swagger.core.version>
- <servlet.api.version>2.5</servlet.api.version>
- <openapi.generator.version>5.0.0</openapi.generator.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
@@ -273,7 +263,7 @@
<version>${javax.xml.bind.version}</version>
</dependency>
<dependency>
- <groupId>org.jline</groupId>
+ <groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>${jline.version}</version>
</dependency>
@@ -288,9 +278,9 @@
<version>1.2.17</version>
</dependency>
<dependency>
- <groupId>org.lz4</groupId>
- <artifactId>lz4-java</artifactId>
- <version>1.8.0</version>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ <version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
@@ -302,10 +292,25 @@
<artifactId>jetty-webapp</artifactId>
<version>${jetty.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
- <version>0.9.2</version>
+ <version>0.7.3</version>
<exclusions>
<!-- This transitive dependency duplicates classes from jline:jline:jar:2.14.5:compile -->
<exclusion>
@@ -329,11 +334,6 @@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-pool2</artifactId>
- <version>${common.pool2.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -391,6 +391,11 @@
<artifactId>javassist</artifactId>
<version>3.24.1-GA</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.10.19</version>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@@ -399,7 +404,7 @@
<dependency>
<groupId>org.objenesis</groupId>
<artifactId>objenesis</artifactId>
- <version>3.0.1</version>
+ <version>3.2</version>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
@@ -409,7 +414,7 @@
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
- <version>2.0.9</version>
+ <version>2.0.2</version>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
@@ -497,7 +502,7 @@
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
- <!-- for cli and test container -->
+ <!-- for test container -->
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
@@ -560,9 +565,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>testcontainers</artifactId>
- <version>1.15.2</version>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
@@ -574,7 +578,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
- <version>2.3</version>
+ <version>2.8.1</version>
<configuration>
<generateBackupPoms>false</generateBackupPoms>
</configuration>
@@ -610,7 +614,7 @@
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
- <version>3.6.1.1688</version>
+ <version>3.9.0.2155</version>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
@@ -668,14 +672,7 @@
<exclude>.pytest_cache/**</exclude>
<exclude>venv/**</exclude>
<exclude>apache_iotdb.egg-info/**</exclude>
- <exclude>**/iotdb/thrift/__init__.py</exclude>
- <exclude>**/iotdb/thrift/rpc/__init__.py</exclude>
- <!-- Java SPI uses files in resources/META-INF/services-->
- <exclude>**/resources/META-INF/services/**</exclude>
- <!-- grafana plugin -->
- <exclude>**/yarn.lock</exclude>
- <!-- site-->
- <exclude>**/redirects</exclude>
+ <exclude>client-py/iotdb/thrift/**</exclude>
</excludes>
</configuration>
</plugin>
@@ -724,9 +721,6 @@
<sourceDirectory>thrift/target/generated-sources/thrift</sourceDirectory>
<sourceDirectory>thrift-sync/target/generated-sources/thrift</sourceDirectory>
<sourceDirectory>thrift-cluster/target/generated-sources/thrift</sourceDirectory>
- <sourceDirectory>thrift-influxdb/target/generated-sources/thrift</sourceDirectory>
- <sourceDirectory>openapi/target/generated-sources/java/src/gen/java</sourceDirectory>
- <sourceDirectory>openapi/target/generated-sources/java/src/main/java</sourceDirectory>
<sourceDirectory>spark-iotdb-connector/src/main/scala</sourceDirectory>
<sourceDirectory>spark-tsfile/src/main/scala</sourceDirectory>
</sourceDirectories>
@@ -736,7 +730,7 @@
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
- <version>2.4.0-b180830.0359</version>
+ <version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
@@ -746,7 +740,7 @@
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
- <version>2.3.6</version>
+ <version>2.3.5</version>
</dependency>
</dependencies>
</plugin>
@@ -827,7 +821,7 @@
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>extra-enforcer-rules</artifactId>
- <version>1.4</version>
+ <version>1.2</version>
</dependency>
</dependencies>
</plugin>
@@ -982,10 +976,6 @@
<!-- Include integration tests within integration-test phase. -->
<include>src/test/**/*IT.java</include>
</includes>
- <excludes>
- <!-- Exclude unit tests within (unit) test phase. -->
- <exclude>src/test/**/*Test.java</exclude>
- </excludes>
</configuration>
</execution>
</executions>
@@ -1005,29 +995,13 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ </plugin>
</plugins>
</build>
<profiles>
- <!-- spotless is too slow, so we put it into a profile to skip it if needed -->
- <!-- currently spotless cannot run on jdk16, due to JEP 396: Strongly Encapsulate JDK Internals by Default-->
- <profile>
- <id>spotless</id>
- <activation>
- <!-- activeByDefault does not take effect-->
- <jdk>[1.8,16)</jdk>
- <file>
- <exists>.</exists>
- </file>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>com.diffplug.spotless</groupId>
- <artifactId>spotless-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
- </profile>
<!--
A set of profiles defining the different properties needed to download and run thrift
They are automatically activated depending on the OS you are using.
@@ -1081,30 +1055,6 @@
<thrift.exec-cmd.args>+x ${project.build.directory}/tools/${thrift.executable}</thrift.exec-cmd.args>
</properties>
</profile>
- <!-- for TestContainer. As it requires docker, we have to detect whether docker exists.-->
- <profile>
- <!-- Mac and Unix-->
- <id>unixDockerCheck</id>
- <activation>
- <file>
- <exists>/var/run/docker.sock</exists>
- </file>
- </activation>
- <modules>
- <module>testcontainer</module>
- </modules>
- </profile>
- <profile>
- <id>WinDockerCheck</id>
- <activation>
- <file>
- <exists>C:\Program Files\Docker\Docker\resources\bin\docker.exe</exists>
- </file>
- </activation>
- <modules>
- <module>testcontainer</module>
- </modules>
- </profile>
<!-- Some APIs were removed in Java 11, so we need to add replacements -->
<profile>
<id>java-11-and-above</id>
@@ -1131,19 +1081,6 @@
</dependency>
</dependencies>
</profile>
- <!-- Add argLine for Java 16 and above, due to [JEP 396: Strongly Encapsulate JDK Internals by Default]
- (https://openjdk.java.net/jeps/396) -->
- <profile>
- <id>java-16-and-above</id>
- <activation>
- <!-- This needs to be updated as soon as Java 20 is shipped -->
- <jdk>[16,20)</jdk>
- </activation>
- <properties>
- <maven.compiler.release>8</maven.compiler.release>
- <argLine>--illegal-access=permit --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.java [...]
- </properties>
- </profile>
<!--
Self activating profile, that activates itself as soon as a "src/main/thrift" directory is found.
The different plugins here download the thrift executable matching the current os, make that
@@ -1168,7 +1105,7 @@
<plugin>
<groupId>com.googlecode.maven-download-plugin</groupId>
<artifactId>download-maven-plugin</artifactId>
- <version>1.6.7</version>
+ <version>1.3.0</version>
<executions>
<execution>
<id>get-thrift-executable</id>
@@ -1344,7 +1281,6 @@
<exclude>org/apache/iotdb/service/sync/thrift/*</exclude>
<exclude>org/apache/iotdb/service/rpc/thrift/*</exclude>
<exclude>org/apache/iotdb/cluster/rpc/thrift/*</exclude>
- <exclude>org/apache/iotdb/protocol/influxdb/rpc/thrift/*</exclude>
<exclude>org/apache/iotdb/db/qp/sql/*</exclude>
</excludes>
<rules>
@@ -1475,7 +1411,6 @@
<execution>
<goals>
<goal>integration-test</goal>
- <goal>verify</goal>
</goals>
</execution>
</executions>
diff --git a/rewriteFileTool/pom.xml b/rewriteFileTool/pom.xml
new file mode 100644
index 0000000000..4a9e65f09e
--- /dev/null
+++ b/rewriteFileTool/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>iotdb-parent</artifactId>
+ <groupId>org.apache.iotdb</groupId>
+ <version>0.12.6-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>rewriteFileTool</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-session</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>client-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assembly/rewriteFileTool.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+</project>
diff --git a/rewriteFileTool/src/assembly/resources/sbin/rewrite-file-tool.sh b/rewriteFileTool/src/assembly/resources/sbin/rewrite-file-tool.sh
new file mode 100644
index 0000000000..988ce60e69
--- /dev/null
+++ b/rewriteFileTool/src/assembly/resources/sbin/rewrite-file-tool.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Rewriting the bad TsFiles
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.RewriteFileTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
diff --git a/rewriteFileTool/src/assembly/resources/sbin/validate-tsfile.sh b/rewriteFileTool/src/assembly/resources/sbin/validate-tsfile.sh
new file mode 100644
index 0000000000..db50b91400
--- /dev/null
+++ b/rewriteFileTool/src/assembly/resources/sbin/validate-tsfile.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Validating the TsFile
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.TsFileValidationTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
\ No newline at end of file
diff --git a/rewriteFileTool/src/assembly/rewriteFileTool.xml b/rewriteFileTool/src/assembly/rewriteFileTool.xml
new file mode 100644
index 0000000000..68b7510885
--- /dev/null
+++ b/rewriteFileTool/src/assembly/rewriteFileTool.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly>
+ <id>RewriteFileTool</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
new file mode 100644
index 0000000000..17b5af7cfb
--- /dev/null
+++ b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * This tool reads tsFiles and rewrites it chunk by chunk. It constructs tablet and invokes
+ * insertTablet() for every page in chunk, in case chunk is too large. `Move` command is used to
+ * unload files in iotdb, and mods files are moved manually.
+ */
+public class RewriteFileTool {
+ // backup data dir path
+ private static String backUpDirPath;
+ // validation file path
+ private static String validationFilePath;
+ // tsfile list path
+ private static String tsfileListPath;
+ // output file path
+ private static String outputLogFilePath;
+
+ private static final String HostIP = "localhost";
+ private static final String rpcPort = "6667";
+ private static final String user = "root";
+ private static final String password = "root";
+
+ private static final FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+
+ private static PrintWriter pw;
+
+ /**
+ * -b=[path of backUp directory] -vf=[path of validation file]/-f=[path of tsfile list] -o=[path
+ * of output log]
+ */
+ public static void main(String[] args) throws IOException {
+ if (!checkArgs(args)) {
+ System.exit(1);
+ }
+ pw = new PrintWriter(new FileWriter(outputLogFilePath));
+ try {
+ if (validationFilePath != null) {
+ readValidationFile(validationFilePath);
+ }
+ if (tsfileListPath != null) {
+ readTsFileList(tsfileListPath);
+ }
+ } catch (Exception e) {
+ printBoth(e.getMessage());
+ e.printStackTrace();
+ } finally {
+ pw.close();
+ }
+ }
+
+ public static void readValidationFile(String validationFilePath)
+ throws IOException, IoTDBConnectionException {
+ Session session = new Session(HostIP, rpcPort, user, password);
+ session.open(false);
+
+ BufferedReader bufferedReader = new BufferedReader(new FileReader(validationFilePath));
+ String line;
+ while ((line = bufferedReader.readLine()) != null) {
+ if (!line.startsWith("-- Find the bad file ")) {
+ continue;
+ }
+ String badFilePath = line.replace("-- Find the bad file ", "");
+ unloadAndReWriteWrongTsFile(badFilePath, session);
+ }
+ bufferedReader.close();
+ session.close();
+ printBoth("Finish rewriting all bad files.");
+ }
+
+ public static void readTsFileList(String tsfileListPath)
+ throws IoTDBConnectionException, IOException {
+ Session session = new Session(HostIP, rpcPort, user, password);
+ session.open(false);
+
+ BufferedReader bufferedReader = new BufferedReader(new FileReader(tsfileListPath));
+ String badFilePath;
+ while ((badFilePath = bufferedReader.readLine()) != null) {
+ unloadAndReWriteWrongTsFile(badFilePath, session);
+ }
+ bufferedReader.close();
+ session.close();
+ printBoth("Finish rewriting all bad files.");
+ }
+
+ public static void unloadAndReWriteWrongTsFile(String filename, Session session) {
+ printBoth(String.format("Start moving %s to backup dir.", filename));
+ try {
+ session.executeNonQueryStatement(String.format("move '%s' '%s'", filename, backUpDirPath));
+ String[] dirs = filename.split("/");
+ String targetFilePath = backUpDirPath + File.separator + dirs[dirs.length - 1];
+ File targetFile = new File(targetFilePath);
+ // move mods file
+ File modsFile = new File(filename + ModificationFile.FILE_SUFFIX);
+ if (modsFile.exists()) {
+ fsFactory.moveFile(modsFile, new File(targetFilePath + ModificationFile.FILE_SUFFIX));
+ }
+ printBoth(String.format("Finish unloading %s.", filename));
+
+ // rewriteFile
+ printBoth(String.format("Start rewriting %s to iotdb.", filename));
+ if (targetFile.exists()) {
+ rewriteWrongTsFile(targetFilePath, session);
+ targetFile.renameTo(new File(targetFilePath + "." + "finish"));
+ } else {
+ printBoth("---- Meet error in rewriting, " + targetFilePath + " does not exist.");
+ }
+ } catch (IoTDBConnectionException | StatementExecutionException e) {
+ e.printStackTrace();
+ printBoth("---- Meet error in unloading " + filename + ", " + e.getMessage());
+ }
+ }
+
+ public static void rewriteWrongTsFile(String filename, Session session) {
+ // read mods file
+ List<Modification> modifications = null;
+ if (FSFactoryProducer.getFSFactory()
+ .getFile(filename + ModificationFile.FILE_SUFFIX)
+ .exists()) {
+ modifications =
+ (List<Modification>)
+ new ModificationFile(filename + ModificationFile.FILE_SUFFIX).getModifications();
+ }
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filename)) {
+ // Sequential reading of one ChunkGroup now follows this order:
+ // first the CHUNK_GROUP_HEADER, then SeriesChunks (headers and data) in one ChunkGroup
+ // Because we do not know how many chunks a ChunkGroup may have, we should read one byte (the
+ // marker) ahead and judge accordingly.
+ reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+ byte marker;
+ String curDevice = null;
+ long chunkHeaderOffset = -1;
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
+ chunkHeaderOffset = reader.position() - 1;
+ ChunkHeader header = reader.readChunkHeader(marker);
+ Decoder defaultTimeDecoder =
+ Decoder.getDecoderByType(
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+ TSDataType.INT64);
+ Decoder valueDecoder =
+ Decoder.getDecoderByType(header.getEncodingType(), header.getDataType());
+ // 1. construct MeasurementSchema from chunkHeader
+ String measurement = header.getMeasurementID();
+ MeasurementSchema measurementSchema =
+ new MeasurementSchema(
+ measurement,
+ header.getDataType(),
+ header.getEncodingType(),
+ header.getCompressionType());
+ // 2. record data point of each measurement
+ int dataSize = header.getDataSize();
+ while (dataSize > 0) {
+ valueDecoder.reset();
+ PageHeader pageHeader =
+ reader.readPageHeader(
+ header.getDataType(), header.getChunkType() == MetaMarker.CHUNK_HEADER);
+ ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
+ PageReader reader1 =
+ new PageReader(
+ pageData, header.getDataType(), valueDecoder, defaultTimeDecoder, null);
+ // read delete time range from old modification file
+ List<TimeRange> deleteIntervalList =
+ getOldSortedDeleteIntervals(
+ curDevice, measurementSchema, chunkHeaderOffset, modifications);
+ reader1.setDeleteIntervalList(deleteIntervalList);
+ BatchData batchData = reader1.getAllSatisfiedPageData();
+ int maxRow;
+ if (header.getChunkType() == MetaMarker.CHUNK_HEADER) {
+ maxRow = (int) pageHeader.getNumOfValues();
+ } else {
+ maxRow = batchData.length();
+ }
+ Tablet tablet =
+ new Tablet(curDevice, Collections.singletonList(measurementSchema), maxRow);
+ int rowIndex = 0;
+ while (batchData.hasCurrent()) {
+ tablet.addTimestamp(rowIndex, batchData.currentTime());
+ tablet.addValue(measurement, rowIndex, batchData.currentValue());
+ batchData.next();
+ rowIndex++;
+ }
+ tablet.rowSize = rowIndex;
+ session.insertTablet(tablet);
+ dataSize -= pageHeader.getSerializedPageSize();
+ }
+ break;
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
+ curDevice = chunkGroupHeader.getDeviceID();
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ reader.readPlanIndex();
+ reader.getMinPlanIndex();
+ reader.getMaxPlanIndex();
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+ } catch (IllegalPathException
+ | IOException
+ | IoTDBConnectionException
+ | StatementExecutionException e) {
+ printBoth("---- Meet error in rewriting " + filename + ", " + e.getMessage());
+ printBoth(e.getMessage());
+ e.printStackTrace();
+ }
+ }
+
+ private static boolean checkArgs(String[] args) {
+ String paramConfig =
+ "-b=[path of backUp directory] -vf=[path of validation file]/-f=[path of tsfile list] -o=[path of output log]";
+ for (String arg : args) {
+ if (arg.startsWith("-b")) {
+ backUpDirPath = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-vf")) {
+ validationFilePath = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-f")) {
+ tsfileListPath = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-o")) {
+ outputLogFilePath = arg.substring(arg.indexOf('=') + 1);
+ } else {
+ System.out.println("Param incorrect!" + paramConfig);
+ return false;
+ }
+ }
+ if (backUpDirPath == null
+ || (validationFilePath == null && tsfileListPath == null)
+ || outputLogFilePath == null) {
+ System.out.println("Param incorrect!" + paramConfig);
+ return false;
+ }
+ return true;
+ }
+
+ private static void printBoth(String msg) {
+ System.out.println(msg);
+ pw.println(msg);
+ }
+
+ private static List<TimeRange> getOldSortedDeleteIntervals(
+ String deviceId,
+ MeasurementSchema schema,
+ long chunkHeaderOffset,
+ List<Modification> modifications)
+ throws IllegalPathException {
+ if (modifications != null && modifications.size() != 0) {
+ Iterator<Modification> modsIterator = modifications.listIterator();
+ ChunkMetadata chunkMetadata = new ChunkMetadata();
+ Deletion currentDeletion = null;
+ while (modsIterator.hasNext()) {
+ currentDeletion = (Deletion) modsIterator.next();
+ // if deletion path match the chunkPath, then add the deletion to the list
+ if (currentDeletion
+ .getPath()
+ .matchFullPath(new PartialPath(deviceId + "." + schema.getMeasurementId()))
+ && currentDeletion.getFileOffset() > chunkHeaderOffset) {
+ chunkMetadata.insertIntoSortedDeletions(
+ currentDeletion.getStartTime(), currentDeletion.getEndTime());
+ }
+ }
+ return chunkMetadata.getDeleteIntervalList();
+ }
+ return null;
+ }
+}
diff --git a/rewriteFileTool/src/main/java/org/apache/iotdb/TsFileValidationTool.java b/rewriteFileTool/src/main/java/org/apache/iotdb/TsFileValidationTool.java
new file mode 100644
index 0000000000..beea1a1075
--- /dev/null
+++ b/rewriteFileTool/src/main/java/org/apache/iotdb/TsFileValidationTool.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
+/**
+ * This tool can be used to check the correctness of tsfile and point out errors in specific
+ * timeseries or devices. The types of errors include the following:
+ *
+ * <p>Device overlap between files
+ *
+ * <p>Timeseries overlap between files
+ *
+ * <p>Timeseries overlap between chunks
+ *
+ * <p>Timeseries overlap between pages
+ *
+ * <p>Timeseries overlap within one page
+ */
+public class TsFileValidationTool {
+ // print detail type of overlap or not
+ private static boolean printDetails = false;
+
+ // print to local file or not
+ private static boolean printToFile = false;
+
+ private static String outFilePath = "TsFile_validation_view.txt";
+
+ private static PrintWriter pw = null;
+
+ private static final Logger logger = LoggerFactory.getLogger(TsFileValidationTool.class);
+ private static final List<File> seqDataDirList = new ArrayList<>();
+ private static final List<File> fileList = new ArrayList<>();
+ private static int badFileNum = 0;
+
+ /**
+ * The form of param is: [path of data dir or tsfile] [-pd = print details or not] [-f = path of
+ * outFile]. Eg: xxx/iotdb/data/data1 xxx/xxx.tsfile -pd=true -f=xxx/TsFile_validation_view.txt
+ *
+ * <p>The first parameter is required, the others are optional.
+ */
+ public static void main(String[] args) throws IOException {
+ if (!checkArgs(args)) {
+ System.exit(1);
+ }
+ if (printToFile) {
+ pw = new PrintWriter(new FileWriter(outFilePath));
+ }
+ if (printDetails) {
+ printBoth("Start checking seq files ...");
+ }
+
+ // check tsfile, which will only check for correctness inside a single tsfile
+ for (File f : fileList) {
+ findUncorrectFiles(Collections.singletonList(f));
+ }
+
+ // check tsfiles in data dir, which will check for correctness inside one single tsfile and
+ // between files
+ for (File seqDataDir : seqDataDirList) {
+ // get sg data dirs
+ if (!checkIsDirectory(seqDataDir)) {
+ continue;
+ }
+ File[] sgDirs = seqDataDir.listFiles();
+ for (File sgDir : Objects.requireNonNull(sgDirs)) {
+ if (!checkIsDirectory(sgDir)) {
+ continue;
+ }
+ if (printDetails) {
+ printBoth("- Check files in storage group: " + sgDir.getAbsolutePath());
+ }
+ // get data region dirs
+ File[] dataRegionDirs = sgDir.listFiles();
+ for (File dataRegionDir : Objects.requireNonNull(dataRegionDirs)) {
+ if (!checkIsDirectory(dataRegionDir)) {
+ continue;
+ }
+ // get time partition dirs and sort them
+ List<File> timePartitionDirs = new ArrayList<>();
+ for (File file : Objects.requireNonNull(dataRegionDir.listFiles())) {
+ if (file != null && !file.getName().endsWith(".DS_Store")) {
+ timePartitionDirs.add(file);
+ }
+ }
+ timePartitionDirs.sort(
+ (f1, f2) ->
+ Long.compareUnsigned(Long.parseLong(f1.getName()), Long.parseLong(f2.getName())));
+ for (File timePartitionDir : timePartitionDirs) {
+ if (!checkIsDirectory(timePartitionDir)) {
+ continue;
+ }
+ // get all seq files under the time partition dir
+ List<File> tsFiles =
+ Arrays.asList(
+ Objects.requireNonNull(
+ timePartitionDir.listFiles(
+ file -> file.getName().endsWith(TSFILE_SUFFIX))));
+ // sort the seq files with timestamp
+ tsFiles.sort(
+ (f1, f2) ->
+ Long.compareUnsigned(
+ Long.parseLong(f1.getName().split("-")[0]),
+ Long.parseLong(f2.getName().split("-")[0])));
+ findUncorrectFiles(tsFiles);
+ }
+ }
+ }
+ }
+ if (printDetails) {
+ printBoth("Finish checking successfully, totally find " + badFileNum + " bad files.");
+ }
+ if (printToFile) {
+ pw.close();
+ }
+ }
+
+ private static void findUncorrectFiles(List<File> tsFiles) {
+ // measurementID -> [lastTime, endTimeInLastFile]
+ Map<String, long[]> measurementLastTime = new HashMap<>();
+ // deviceID -> endTime, the endTime of device in the last seq file
+ Map<String, Long> deviceEndTime = new HashMap<>();
+
+ for (File tsFile : tsFiles) {
+ try {
+ TsFileResource resource = new TsFileResource(tsFile);
+ if (!new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()) {
+ // resource file does not exist, tsfile may not be flushed yet
+ logger.warn(
+ "{} does not exist ,skip it.",
+ tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX);
+ continue;
+ } else {
+ resource.deserialize();
+ }
+ boolean isBadFile = false;
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
+ // deviceID -> has checked overlap or not
+ Map<String, Boolean> hasCheckedDeviceOverlap = new HashMap<>();
+ reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+ byte marker;
+ String deviceID = "";
+ Map<String, boolean[]> hasMeasurementPrintedDetails = new HashMap<>();
+ // measurementId -> lastChunkEndTime in current file
+ Map<String, Long> lashChunkEndTime = new HashMap<>();
+
+ // start reading data points in sequence
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
+ ChunkHeader header = reader.readChunkHeader(marker);
+ if (header.getDataSize() == 0) {
+ // empty value chunk
+ break;
+ }
+ long currentChunkEndTime = Long.MIN_VALUE;
+ String measurementID = deviceID + "." + header.getMeasurementID();
+ hasMeasurementPrintedDetails.computeIfAbsent(measurementID, k -> new boolean[4]);
+ measurementLastTime.computeIfAbsent(
+ measurementID,
+ k -> {
+ long[] arr = new long[2];
+ Arrays.fill(arr, Long.MIN_VALUE);
+ return arr;
+ });
+ Decoder defaultTimeDecoder =
+ Decoder.getDecoderByType(
+ TSEncoding.valueOf(
+ TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+ TSDataType.INT64);
+ Decoder valueDecoder =
+ Decoder.getDecoderByType(header.getEncodingType(), header.getDataType());
+ int dataSize = header.getDataSize();
+ long lastPageEndTime = Long.MIN_VALUE;
+ while (dataSize > 0) {
+ valueDecoder.reset();
+ PageHeader pageHeader =
+ reader.readPageHeader(
+ header.getDataType(),
+ (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER);
+ ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
+ long currentPageEndTime = Long.MIN_VALUE;
+ // NonAligned Chunk
+ PageReader pageReader =
+ new PageReader(
+ pageData, header.getDataType(), valueDecoder, defaultTimeDecoder, null);
+ BatchData batchData = pageReader.getAllSatisfiedPageData();
+ while (batchData.hasCurrent()) {
+ long timestamp = batchData.currentTime();
+ if (timestamp <= measurementLastTime.get(measurementID)[0]) {
+ // find bad file
+ if (!isBadFile) {
+ if (printDetails) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ } else {
+ printBoth(tsFile.getAbsolutePath());
+ }
+ isBadFile = true;
+ badFileNum++;
+ }
+ if (printDetails) {
+ if (timestamp <= measurementLastTime.get(measurementID)[1]) {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[0]) {
+ printBoth(
+ "-------- Timeseries " + measurementID + " overlap between files");
+ hasMeasurementPrintedDetails.get(measurementID)[0] = true;
+ }
+ } else if (timestamp
+ <= lashChunkEndTime.getOrDefault(measurementID, Long.MIN_VALUE)) {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[1]) {
+ printBoth(
+ "-------- Timeseries " + measurementID + " overlap between chunks");
+ hasMeasurementPrintedDetails.get(measurementID)[1] = true;
+ }
+ } else if (timestamp <= lastPageEndTime) {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[2]) {
+ printBoth(
+ "-------- Timeseries " + measurementID + " overlap between pages");
+ hasMeasurementPrintedDetails.get(measurementID)[2] = true;
+ }
+ } else {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[3]) {
+ printBoth(
+ "-------- Timeseries "
+ + measurementID
+ + " overlap within one page");
+ hasMeasurementPrintedDetails.get(measurementID)[3] = true;
+ }
+ }
+ }
+ } else {
+ measurementLastTime.get(measurementID)[0] = timestamp;
+ currentPageEndTime = timestamp;
+ currentChunkEndTime = timestamp;
+ }
+ batchData.next();
+ }
+ dataSize -= pageHeader.getSerializedPageSize();
+ lastPageEndTime = Math.max(lastPageEndTime, currentPageEndTime);
+ }
+ lashChunkEndTime.put(
+ measurementID,
+ Math.max(
+ lashChunkEndTime.getOrDefault(measurementID, Long.MIN_VALUE),
+ currentChunkEndTime));
+ break;
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ if (!deviceID.equals("")) {
+ // record the end time of last device in current file
+ if (resource.getEndTime(deviceID)
+ > deviceEndTime.getOrDefault(deviceID, Long.MIN_VALUE)) {
+ deviceEndTime.put(deviceID, resource.getEndTime(deviceID));
+ }
+ }
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
+ deviceID = chunkGroupHeader.getDeviceID();
+ if (!hasCheckedDeviceOverlap.getOrDefault(deviceID, false)
+ && resource.getStartTime(deviceID)
+ <= deviceEndTime.getOrDefault(deviceID, Long.MIN_VALUE)) {
+ // find bad file
+ if (!isBadFile) {
+ if (printDetails) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ } else {
+ printBoth(tsFile.getAbsolutePath());
+ }
+ isBadFile = true;
+ badFileNum++;
+ }
+ if (printDetails) {
+ printBoth("---- Device " + deviceID + " overlap between files");
+ }
+ }
+ hasCheckedDeviceOverlap.put(deviceID, true);
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ reader.readPlanIndex();
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+
+ // record the end time of each timeseries in current file
+ for (Map.Entry<String, Long> entry : lashChunkEndTime.entrySet()) {
+ Long endTime = Math.max(measurementLastTime.get(entry.getKey())[1], entry.getValue());
+ measurementLastTime.get(entry.getKey())[1] = endTime;
+ }
+ }
+ } catch (Throwable e) {
+ logger.error("Meet errors in reading file {} , skip it.", tsFile.getAbsolutePath(), e);
+ if (printDetails) {
+ printBoth("-- Meet errors in reading file " + tsFile.getAbsolutePath());
+ }
+ }
+ }
+ }
+
+ public static boolean checkArgs(String[] args) {
+ if (args.length < 1) {
+ System.out.println(
+ "Please input correct param, which is [path of data dir] [-pd = print details or not] [-f = path of outFile]. Eg: xxx/iotdb/data/data -pd=true -f=xxx/TsFile_validation_view.txt");
+ return false;
+ } else {
+ for (String arg : args) {
+ if (arg.startsWith("-pd")) {
+ printDetails = Boolean.parseBoolean(arg.split("=")[1]);
+ } else if (arg.startsWith("-f")) {
+ printToFile = true;
+ outFilePath = arg.split("=")[1];
+ } else {
+ File f = new File(arg);
+ if (f.isDirectory()
+ && Objects.requireNonNull(
+ f.list(
+ (dir, name) ->
+ (name.equals("sequence") || name.equals("unsequence"))))
+ .length
+ == 2) {
+ File seqDataDir = new File(f, "sequence");
+ seqDataDirList.add(seqDataDir);
+ } else if (arg.endsWith(TSFILE_SUFFIX) && f.isFile()) {
+ fileList.add(f);
+ } else {
+ System.out.println(arg + " is not a correct data directory or tsfile of IOTDB.");
+ return false;
+ }
+ }
+ }
+ if (seqDataDirList.size() == 0 && fileList.size() == 0) {
+ System.out.println(
+ "Please input correct param, which is [path of data dir] [-pd = print details or not] [-f = path of outFile]. Eg: xxx/iotdb/data/data -pd=true -f=xxx/TsFile_validation_view.txt");
+ return false;
+ }
+ return true;
+ }
+ }
+
+ private static boolean checkIsDirectory(File dir) {
+ boolean res = true;
+ if (!dir.isDirectory()) {
+ logger.error("{} is not a directory or does not exist, skip it.", dir.getAbsolutePath());
+ res = false;
+ }
+ return res;
+ }
+
+ private static void printBoth(String msg) {
+ System.out.println(msg);
+ if (printToFile) {
+ pw.println(msg);
+ }
+ }
+}