You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/05/26 02:42:44 UTC
[iotdb] branch rewriteTool created (now 2bc26b539e)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 2bc26b539e add rename function
This branch includes the following new commits:
new 929f0c4e3b add rewrite tool
new 2bc26b539e add rename function
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: add rename function
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2bc26b539e2829b948ea1990f12b4d30e84e6c82
Author: Alima777 <wx...@gmail.com>
AuthorDate: Wed May 25 19:57:04 2022 +0800
add rename function
---
rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
index 9a75e529c6..a33ccbfafd 100644
--- a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
+++ b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
@@ -144,8 +144,10 @@ public class RewriteBadFileTool {
String targetFilePath =
backUpDirPath + File.separator + "sequence" + badFilePath.split("sequence")[1];
try {
- if (new File(targetFilePath).exists()) {
+ File targetFile = new File(targetFilePath);
+ if (targetFile.exists()) {
rewriteWrongTsFile(targetFilePath, session);
+ targetFile.renameTo(new File(targetFilePath + "." + "finish"));
} else {
printBoth("---- Meet error in rewriting, " + targetFilePath + " does not exist.");
}
[iotdb] 01/02: add rewrite tool
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 929f0c4e3bc5e5ab62039a634b659c5a813f2995
Author: Alima777 <wx...@gmail.com>
AuthorDate: Wed May 25 16:24:21 2022 +0800
add rewrite tool
---
.../org/apache/iotdb/db/qp/sql/SqlLexer.tokens | 253 ++++++++++++++
distribution/pom.xml | 5 +-
distribution/src/assembly/rewriteBadFile.xml | 49 +++
pom.xml | 1 +
rewriteBadFile/pom.xml | 62 ++++
.../resources/sbin/rewrite-badFile-tool.sh | 48 +++
.../src/assembly/resources/sbin/validate-tsfile.sh | 48 +++
rewriteBadFile/src/assembly/rewriteBadFile.xml | 40 +++
.../java/org/apache/iotdb/RewriteBadFileTool.java | 310 +++++++++++++++++
.../org/apache/iotdb/TsFileValidationTool.java | 379 +++++++++++++++++++++
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 2 +-
11 files changed, 1194 insertions(+), 3 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.tokens b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.tokens
new file mode 100644
index 0000000000..bc327191f9
--- /dev/null
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.tokens
@@ -0,0 +1,253 @@
+WS=1
+ADD=2
+AFTER=3
+ALIAS=4
+ALIGN=5
+ALIGNED=6
+ALL=7
+ALTER=8
+ANY=9
+APPEND=10
+AS=11
+ASC=12
+ATTRIBUTES=13
+AUTOREGISTER=14
+BEFORE=15
+BEGIN=16
+BOUNDARY=17
+BY=18
+CACHE=19
+CHILD=20
+CLEAR=21
+COMPRESSION=22
+COMPRESSOR=23
+CONCAT=24
+CONFIGURATION=25
+CONTINUOUS=26
+COUNT=27
+CONTAIN=28
+CQ=29
+CQS=30
+CREATE=31
+DATATYPE=32
+DEBUG=33
+DELETE=34
+DESC=35
+DESCRIBE=36
+DEVICE=37
+DEVICES=38
+DISABLE=39
+DROP=40
+ENCODING=41
+END=42
+EVERY=43
+EXPLAIN=44
+FILL=45
+FLUSH=46
+FOR=47
+FROM=48
+FULL=49
+FUNCTION=50
+FUNCTIONS=51
+GLOBAL=52
+GRANT=53
+GROUP=54
+INDEX=55
+INFO=56
+INSERT=57
+INTO=58
+KILL=59
+LABEL=60
+LAST=61
+LATEST=62
+LEVEL=63
+LIKE=64
+LIMIT=65
+LINEAR=66
+LINK=67
+LIST=68
+LOAD=69
+LOCK=70
+MERGE=71
+METADATA=72
+NODES=73
+NOW=74
+OF=75
+OFF=76
+OFFSET=77
+ON=78
+ORDER=79
+PARTITION=80
+PASSWORD=81
+PATHS=82
+PIPE=83
+PIPES=84
+PIPESERVER=85
+PIPESINK=86
+PIPESINKS=87
+PIPESINKTYPE=88
+PREVIOUS=89
+PREVIOUSUNTILLAST=90
+PRIVILEGES=91
+PROCESSLIST=92
+PROPERTY=93
+PRUNE=94
+QUERIES=95
+QUERY=96
+READONLY=97
+REGEXP=98
+REMOVE=99
+RENAME=100
+RESAMPLE=101
+RESOURCE=102
+REVOKE=103
+ROLE=104
+ROOT=105
+SCHEMA=106
+SELECT=107
+SET=108
+SETTLE=109
+SGLEVEL=110
+SHOW=111
+SLIMIT=112
+SOFFSET=113
+STORAGE=114
+START=115
+STOP=116
+SYSTEM=117
+TAGS=118
+TASK=119
+TEMPLATE=120
+TEMPLATES=121
+TIME=122
+TIMESERIES=123
+TIMESTAMP=124
+TO=125
+TOLERANCE=126
+TOP=127
+TRACING=128
+TRIGGER=129
+TRIGGERS=130
+TTL=131
+UNLINK=132
+UNLOAD=133
+UNSET=134
+UPDATE=135
+UPSERT=136
+USER=137
+USING=138
+VALUES=139
+VERIFY=140
+VERSION=141
+WATERMARK_EMBEDDING=142
+WHERE=143
+WITH=144
+WITHOUT=145
+WRITABLE=146
+DATATYPE_VALUE=147
+BOOLEAN=148
+DOUBLE=149
+FLOAT=150
+INT32=151
+INT64=152
+TEXT=153
+ENCODING_VALUE=154
+DICTIONARY=155
+DIFF=156
+GORILLA=157
+PLAIN=158
+REGULAR=159
+RLE=160
+TS_2DIFF=161
+ZIGZAG=162
+FREQ=163
+COMPRESSOR_VALUE=164
+GZIP=165
+LZ4=166
+SNAPPY=167
+UNCOMPRESSED=168
+PRIVILEGE_VALUE=169
+SET_STORAGE_GROUP=170
+DELETE_STORAGE_GROUP=171
+CREATE_TIMESERIES=172
+INSERT_TIMESERIES=173
+READ_TIMESERIES=174
+DELETE_TIMESERIES=175
+CREATE_USER=176
+DELETE_USER=177
+MODIFY_PASSWORD=178
+LIST_USER=179
+GRANT_USER_PRIVILEGE=180
+REVOKE_USER_PRIVILEGE=181
+GRANT_USER_ROLE=182
+REVOKE_USER_ROLE=183
+CREATE_ROLE=184
+DELETE_ROLE=185
+LIST_ROLE=186
+GRANT_ROLE_PRIVILEGE=187
+REVOKE_ROLE_PRIVILEGE=188
+CREATE_FUNCTION=189
+DROP_FUNCTION=190
+CREATE_TRIGGER=191
+DROP_TRIGGER=192
+START_TRIGGER=193
+STOP_TRIGGER=194
+CREATE_CONTINUOUS_QUERY=195
+DROP_CONTINUOUS_QUERY=196
+SCHEMA_REPLICATION_FACTOR=197
+DATA_REPLICATION_FACTOR=198
+TIME_PARTITION_INTERVAL=199
+MINUS=200
+PLUS=201
+DIV=202
+MOD=203
+OPERATOR_DEQ=204
+OPERATOR_SEQ=205
+OPERATOR_GT=206
+OPERATOR_GTE=207
+OPERATOR_LT=208
+OPERATOR_LTE=209
+OPERATOR_NEQ=210
+OPERATOR_IN=211
+OPERATOR_AND=212
+OPERATOR_OR=213
+OPERATOR_NOT=214
+OPERATOR_CONTAINS=215
+DOT=216
+COMMA=217
+SEMI=218
+STAR=219
+DOUBLE_STAR=220
+LR_BRACKET=221
+RR_BRACKET=222
+LS_BRACKET=223
+RS_BRACKET=224
+STRING_LITERAL=225
+DURATION_LITERAL=226
+DATETIME_LITERAL=227
+INTEGER_LITERAL=228
+EXPONENT_NUM_PART=229
+BOOLEAN_LITERAL=230
+NULL_LITERAL=231
+NAN_LITERAL=232
+ID=233
+QUOTED_ID=234
+'-'=200
+'+'=201
+'/'=202
+'%'=203
+'=='=204
+'='=205
+'>'=206
+'>='=207
+'<'=208
+'<='=209
+'.'=216
+','=217
+';'=218
+'*'=219
+'**'=220
+'('=221
+')'=222
+'['=223
+']'=224
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 67bb6467c7..66c7816e6c 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -52,6 +52,7 @@
<descriptor>src/assembly/cli.xml</descriptor>
<descriptor>src/assembly/grafana.xml</descriptor>
<descriptor>src/assembly/client-cpp.xml</descriptor>
+ <descriptor>src/assembly/rewriteBadFile.xml</descriptor>
</descriptors>
<finalName>apache-iotdb-${project.version}</finalName>
</configuration>
@@ -59,8 +60,8 @@
</executions>
</plugin>
<!--
- Create SHA512 checksum files for the release artifacts.
- -->
+Create SHA512 checksum files for the release artifacts.
+-->
<plugin>
<groupId>net.nicoulaj.maven.plugins</groupId>
<artifactId>checksum-maven-plugin</artifactId>
diff --git a/distribution/src/assembly/rewriteBadFile.xml b/distribution/src/assembly/rewriteBadFile.xml
new file mode 100644
index 0000000000..faddb91ee1
--- /dev/null
+++ b/distribution/src/assembly/rewriteBadFile.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly>
+ <id>cli-bin</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <baseDirectory>apache-iotdb-${project.version}-rewriteBadFile-bin</baseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <includes>
+ <include>*:iotdb-rewriteBadFile:zip</include>
+ </includes>
+ <outputDirectory>${file.separator}</outputDirectory>
+ <outputFileNameMapping>${artifact.artifactId}.${artifact.extension}</outputFileNameMapping>
+ <unpack>true</unpack>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <outputDirectory>sbin</outputDirectory>
+ <directory>${maven.multiModuleProjectDirectory}/rewriteBadFile/src/assembly/resources/sbin</directory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ </fileSets>
+ <componentDescriptors>
+ <componentDescriptor>common-files.xml</componentDescriptor>
+ </componentDescriptors>
+</assembly>
diff --git a/pom.xml b/pom.xml
index 5318a29530..7e0242c8b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -104,6 +104,7 @@
<module>client-py</module>
<module>compile-tools</module>
<module>client-cpp</module>
+ <module>rewriteBadFile</module>
</modules>
<!-- Properties Management -->
<properties>
diff --git a/rewriteBadFile/pom.xml b/rewriteBadFile/pom.xml
new file mode 100644
index 0000000000..7953a5e9e5
--- /dev/null
+++ b/rewriteBadFile/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>iotdb-parent</artifactId>
+ <groupId>org.apache.iotdb</groupId>
+ <version>0.12.6-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>rewriteBadFile</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-session</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>client-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assembly/rewriteBadFile.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <properties>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ </properties>
+</project>
diff --git a/rewriteBadFile/src/assembly/resources/sbin/rewrite-badFile-tool.sh b/rewriteBadFile/src/assembly/resources/sbin/rewrite-badFile-tool.sh
new file mode 100644
index 0000000000..4f0d45c931
--- /dev/null
+++ b/rewriteBadFile/src/assembly/resources/sbin/rewrite-badFile-tool.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Rewriting the bad TsFiles
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.RewriteBadFileTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
diff --git a/rewriteBadFile/src/assembly/resources/sbin/validate-tsfile.sh b/rewriteBadFile/src/assembly/resources/sbin/validate-tsfile.sh
new file mode 100644
index 0000000000..db50b91400
--- /dev/null
+++ b/rewriteBadFile/src/assembly/resources/sbin/validate-tsfile.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Validating the TsFile
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.TsFileValidationTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
\ No newline at end of file
diff --git a/rewriteBadFile/src/assembly/rewriteBadFile.xml b/rewriteBadFile/src/assembly/rewriteBadFile.xml
new file mode 100644
index 0000000000..dfbd548135
--- /dev/null
+++ b/rewriteBadFile/src/assembly/rewriteBadFile.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly>
+ <id>rewriteBadFile</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
new file mode 100644
index 0000000000..9a75e529c6
--- /dev/null
+++ b/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+public class RewriteBadFileTool {
+ // backup data dir path
+ private static String backUpDirPath = "backup";
+ // validation file path
+ private static String validationFilePath = "TsFile_validation_view.txt";
+ // output file path
+ private static String outputLogFilePath = "TsFile_rewrite_view.txt";
+ // whether moving the data files
+ private static boolean moveFile = true;
+
+ private static final String HostIP = "localhost";
+ private static final String rpcPort = "6667";
+ private static final String user = "root";
+ private static final String password = "root";
+
+ private static final FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+
+ private static PrintWriter pw;
+
+ /**
+ * -b=[path of backUp directory] -v=[path of validation file] -o=[path of output log] -m=[whether
+ * move the file]
+ */
+ public static void main(String[] args) throws IOException {
+ if (!checkArgs(args)) {
+ System.exit(1);
+ }
+ pw = new PrintWriter(new FileWriter(outputLogFilePath));
+ try {
+ if (moveFile) {
+ moveBadFileToBackUp();
+ } else {
+ rewriteAllBadFiles();
+ }
+ } catch (IoTDBConnectionException | IOException e) {
+ e.printStackTrace();
+ } finally {
+ pw.close();
+ }
+ }
+
+ public static void moveBadFileToBackUp() throws IOException {
+ printBoth("Start moving bad files to backup dir.");
+ BufferedReader bufferedReader = new BufferedReader(new FileReader(validationFilePath));
+ String line;
+ while ((line = bufferedReader.readLine()) != null) {
+ if (!line.startsWith("-- Find the bad file ")) {
+ continue;
+ }
+ String badFilePath = line.replace("-- Find the bad file ", "");
+ String targetFilePath =
+ backUpDirPath + File.separator + "sequence" + badFilePath.split("sequence")[1];
+ File targetFile = new File(targetFilePath);
+ if (!targetFile.getParentFile().exists()) {
+ targetFile.getParentFile().mkdir();
+ }
+ // move tsfile
+ fsFactory.moveFile(new File(badFilePath), targetFile);
+ // move resource file
+ fsFactory.moveFile(
+ new File(badFilePath + TsFileResource.RESOURCE_SUFFIX),
+ new File(targetFilePath + TsFileResource.RESOURCE_SUFFIX));
+ // move mods file
+ File modsFile = new File(badFilePath + ModificationFile.FILE_SUFFIX);
+ if (modsFile.exists()) {
+ fsFactory.moveFile(modsFile, new File(targetFilePath + ModificationFile.FILE_SUFFIX));
+ }
+ }
+ bufferedReader.close();
+ printBoth("Finish moving all bad files to backup dir.");
+ }
+
+ private static void rewriteAllBadFiles() throws IOException, IoTDBConnectionException {
+ printBoth("Start rewriting bad files to iotdb.");
+ Session session = new Session(HostIP, rpcPort, user, password);
+ session.open(false);
+ BufferedReader bufferedReader = new BufferedReader(new FileReader(validationFilePath));
+ String line;
+ while ((line = bufferedReader.readLine()) != null) {
+ if (!line.startsWith("-- Find the bad file ")) {
+ continue;
+ }
+ String badFilePath = line.replace("-- Find the bad file ", "");
+ String targetFilePath =
+ backUpDirPath + File.separator + "sequence" + badFilePath.split("sequence")[1];
+ try {
+ if (new File(targetFilePath).exists()) {
+ rewriteWrongTsFile(targetFilePath, session);
+ } else {
+ printBoth("---- Meet error in rewriting, " + targetFilePath + " does not exist.");
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ printBoth("---- Meet error in rewriting " + targetFilePath + ", " + e.getMessage());
+ }
+ }
+ session.close();
+ printBoth("Finish rewriting all bad files to iotdb.");
+ }
+
+ public static void rewriteWrongTsFile(String filename, Session session)
+ throws IoTDBConnectionException, StatementExecutionException, IOException,
+ IllegalPathException {
+ // read mods file
+ List<Modification> modifications = null;
+ if (FSFactoryProducer.getFSFactory()
+ .getFile(filename + ModificationFile.FILE_SUFFIX)
+ .exists()) {
+ modifications =
+ (List<Modification>)
+ new ModificationFile(filename + ModificationFile.FILE_SUFFIX).getModifications();
+ }
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filename)) {
+ // Sequential reading of one ChunkGroup now follows this order:
+ // first the CHUNK_GROUP_HEADER, then SeriesChunks (headers and data) in one ChunkGroup
+ // Because we do not know how many chunks a ChunkGroup may have, we should read one byte (the
+ // marker) ahead and judge accordingly.
+ reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+ byte marker;
+ String curDevice = null;
+ long chunkHeaderOffset = -1;
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
+ chunkHeaderOffset = reader.position() - 1;
+ ChunkHeader header = reader.readChunkHeader(marker);
+ Decoder defaultTimeDecoder =
+ Decoder.getDecoderByType(
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+ TSDataType.INT64);
+ Decoder valueDecoder =
+ Decoder.getDecoderByType(header.getEncodingType(), header.getDataType());
+ // 1. construct MeasurementSchema from chunkHeader
+ String measurement = header.getMeasurementID();
+ MeasurementSchema measurementSchema =
+ new MeasurementSchema(
+ measurement,
+ header.getDataType(),
+ header.getEncodingType(),
+ header.getCompressionType());
+ // 2. record data point of each measurement
+ int dataSize = header.getDataSize();
+ while (dataSize > 0) {
+ valueDecoder.reset();
+ PageHeader pageHeader =
+ reader.readPageHeader(
+ header.getDataType(), header.getChunkType() == MetaMarker.CHUNK_HEADER);
+ ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
+ PageReader reader1 =
+ new PageReader(
+ pageData, header.getDataType(), valueDecoder, defaultTimeDecoder, null);
+ // read delete time range from old modification file
+ List<TimeRange> deleteIntervalList =
+ getOldSortedDeleteIntervals(
+ curDevice, measurementSchema, chunkHeaderOffset, modifications);
+ reader1.setDeleteIntervalList(deleteIntervalList);
+ BatchData batchData = reader1.getAllSatisfiedPageData();
+ int maxRow;
+ if (header.getChunkType() == MetaMarker.CHUNK_HEADER) {
+ maxRow = (int) pageHeader.getNumOfValues();
+ } else {
+ maxRow = batchData.length();
+ }
+ Tablet tablet =
+ new Tablet(curDevice, Collections.singletonList(measurementSchema), maxRow);
+ int rowIndex = 0;
+ while (batchData.hasCurrent()) {
+ tablet.addTimestamp(rowIndex, batchData.currentTime());
+ tablet.addValue(measurement, rowIndex, batchData.currentValue());
+ batchData.next();
+ rowIndex++;
+ }
+ tablet.rowSize = rowIndex;
+ session.insertTablet(tablet);
+ dataSize -= pageHeader.getSerializedPageSize();
+ }
+ break;
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
+ curDevice = chunkGroupHeader.getDeviceID();
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ reader.readPlanIndex();
+ reader.getMinPlanIndex();
+ reader.getMaxPlanIndex();
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+ }
+ }
+
+ private static boolean checkArgs(String[] args) {
+ if (args.length != 4) {
+ System.out.println(
+ "Param incorrect, -m=[need moving data file or not] -b=[path of backUp directory] -v=[path of validation file] -o=[path of output file].");
+ return false;
+ }
+ for (String arg : args) {
+ if (arg.startsWith("-b")) {
+ backUpDirPath = arg.split("=")[1];
+ } else if (arg.startsWith("-v")) {
+ validationFilePath = arg.split("=")[1];
+ } else if (arg.startsWith("-o")) {
+ outputLogFilePath = arg.split("=")[1];
+ } else if (arg.startsWith("-m")) {
+ moveFile = Boolean.parseBoolean(arg.split("=")[1]);
+ } else {
+ System.out.println(
+ "Param incorrect, -m=[need moving data file or not] -b=[path of backUp directory] -v=[path of validation file] -o=[path of output file].");
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static void printBoth(String msg) {
+ System.out.println(msg);
+ pw.println(msg);
+ }
+
+ private static List<TimeRange> getOldSortedDeleteIntervals(
+ String deviceId,
+ MeasurementSchema schema,
+ long chunkHeaderOffset,
+ List<Modification> modifications)
+ throws IllegalPathException {
+ if (modifications != null && modifications.size() != 0) {
+ Iterator<Modification> modsIterator = modifications.listIterator();
+ ChunkMetadata chunkMetadata = new ChunkMetadata();
+ Deletion currentDeletion = null;
+ while (modsIterator.hasNext()) {
+ currentDeletion = (Deletion) modsIterator.next();
+ // if deletion path match the chunkPath, then add the deletion to the list
+ if (currentDeletion
+ .getPath()
+ .matchFullPath(new PartialPath(deviceId + "." + schema.getMeasurementId()))
+ && currentDeletion.getFileOffset() > chunkHeaderOffset) {
+ chunkMetadata.insertIntoSortedDeletions(
+ currentDeletion.getStartTime(), currentDeletion.getEndTime());
+ }
+ }
+ return chunkMetadata.getDeleteIntervalList();
+ }
+ return null;
+ }
+}
diff --git a/rewriteBadFile/src/main/java/org/apache/iotdb/TsFileValidationTool.java b/rewriteBadFile/src/main/java/org/apache/iotdb/TsFileValidationTool.java
new file mode 100644
index 0000000000..863c3c41ef
--- /dev/null
+++ b/rewriteBadFile/src/main/java/org/apache/iotdb/TsFileValidationTool.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
+/**
+ * This tool can be used to check the correctness of tsfile and point out errors in specific
+ * timeseries or devices. The types of errors include the following:
+ *
+ * <p>Device overlap between files
+ *
+ * <p>Timeseries overlap between files
+ *
+ * <p>Timeseries overlap between chunks
+ *
+ * <p>Timeseries overlap between pages
+ *
+ * <p>Timeseries overlap within one page
+ */
+public class TsFileValidationTool {
+ // print detail type of overlap or not
+ private static boolean printDetails = false;
+
+ // print to local file or not
+ private static boolean printToFile = false;
+
+ private static String outFilePath = "TsFile_validation_view.txt";
+
+ private static PrintWriter pw = null;
+
+ private static final Logger logger = LoggerFactory.getLogger(TsFileValidationTool.class);
+ private static final List<File> seqDataDirList = new ArrayList<>();
+ private static final List<File> fileList = new ArrayList<>();
+ private static int badFileNum = 0;
+
+ /**
+ * The form of param is: [path of data dir or tsfile] [-pd = print details or not] [-f = path of
+ * outFile]. Eg: xxx/iotdb/data/data1 xxx/xxx.tsfile -pd=true -f=xxx/TsFile_validation_view.txt
+ *
+ * <p>The first parameter is required, the others are optional.
+ */
+ public static void main(String[] args) throws IOException {
+ if (!checkArgs(args)) {
+ System.exit(1);
+ }
+ if (printToFile) {
+ pw = new PrintWriter(new FileWriter(outFilePath));
+ }
+ printBoth("Start checking seq files ...");
+
+ // check tsfile, which will only check for correctness inside a single tsfile
+ for (File f : fileList) {
+ findUncorrectFiles(Collections.singletonList(f));
+ }
+
+ // check tsfiles in data dir, which will check for correctness inside one single tsfile and
+ // between files
+ for (File seqDataDir : seqDataDirList) {
+ // get sg data dirs
+ if (!checkIsDirectory(seqDataDir)) {
+ continue;
+ }
+ File[] sgDirs = seqDataDir.listFiles();
+ for (File sgDir : Objects.requireNonNull(sgDirs)) {
+ if (!checkIsDirectory(sgDir)) {
+ continue;
+ }
+ printBoth("- Check files in storage group: " + sgDir.getAbsolutePath());
+ // get data region dirs
+ File[] dataRegionDirs = sgDir.listFiles();
+ for (File dataRegionDir : Objects.requireNonNull(dataRegionDirs)) {
+ if (!checkIsDirectory(dataRegionDir)) {
+ continue;
+ }
+ // get time partition dir
+ File[] timePartitionDirs = dataRegionDir.listFiles();
+ for (File timePartitionDir : Objects.requireNonNull(timePartitionDirs)) {
+ if (!checkIsDirectory(timePartitionDir)) {
+ continue;
+ }
+ // get all seq files under the time partition dir
+ List<File> tsFiles =
+ Arrays.asList(
+ Objects.requireNonNull(
+ timePartitionDir.listFiles(
+ file -> file.getName().endsWith(TSFILE_SUFFIX))));
+ // sort the seq files with timestamp
+ tsFiles.sort(
+ (f1, f2) ->
+ Long.compareUnsigned(
+ Long.parseLong(f1.getName().split("-")[0]),
+ Long.parseLong(f2.getName().split("-")[0])));
+ findUncorrectFiles(tsFiles);
+ }
+ }
+ }
+ }
+ printBoth("Finish checking successfully, totally find " + badFileNum + " bad files.");
+ if (printToFile) {
+ pw.close();
+ }
+ }
+
+ private static void findUncorrectFiles(List<File> tsFiles) {
+ // measurementID -> [lastTime, endTimeInLastFile]
+ Map<String, long[]> measurementLastTime = new HashMap<>();
+ // deviceID -> endTime, the endTime of device in the last seq file
+ Map<String, Long> deviceEndTime = new HashMap<>();
+
+ for (File tsFile : tsFiles) {
+ try {
+ TsFileResource resource = new TsFileResource(tsFile);
+ if (!new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()) {
+ // resource file does not exist, tsfile may not be flushed yet
+ logger.warn(
+ "{} does not exist ,skip it.",
+ tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX);
+ continue;
+ } else {
+ resource.deserialize();
+ }
+ boolean isBadFile = false;
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
+ // deviceID -> has checked overlap or not
+ Map<String, Boolean> hasCheckedDeviceOverlap = new HashMap<>();
+ reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+ byte marker;
+ String deviceID = "";
+ Map<String, boolean[]> hasMeasurementPrintedDetails = new HashMap<>();
+ // measurementId -> lastChunkEndTime in current file
+ Map<String, Long> lashChunkEndTime = new HashMap<>();
+
+ // start reading data points in sequence
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
+ ChunkHeader header = reader.readChunkHeader(marker);
+ if (header.getDataSize() == 0) {
+ // empty value chunk
+ break;
+ }
+ long currentChunkEndTime = Long.MIN_VALUE;
+ String measurementID = deviceID + "." + header.getMeasurementID();
+ hasMeasurementPrintedDetails.computeIfAbsent(measurementID, k -> new boolean[4]);
+ measurementLastTime.computeIfAbsent(
+ measurementID,
+ k -> {
+ long[] arr = new long[2];
+ Arrays.fill(arr, Long.MIN_VALUE);
+ return arr;
+ });
+ Decoder defaultTimeDecoder =
+ Decoder.getDecoderByType(
+ TSEncoding.valueOf(
+ TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+ TSDataType.INT64);
+ Decoder valueDecoder =
+ Decoder.getDecoderByType(header.getEncodingType(), header.getDataType());
+ int dataSize = header.getDataSize();
+ long lastPageEndTime = Long.MIN_VALUE;
+ while (dataSize > 0) {
+ valueDecoder.reset();
+ PageHeader pageHeader =
+ reader.readPageHeader(
+ header.getDataType(),
+ (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER);
+ ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
+ long currentPageEndTime = Long.MIN_VALUE;
+ // NonAligned Chunk
+ PageReader pageReader =
+ new PageReader(
+ pageData, header.getDataType(), valueDecoder, defaultTimeDecoder, null);
+ BatchData batchData = pageReader.getAllSatisfiedPageData();
+ while (batchData.hasCurrent()) {
+ long timestamp = batchData.currentTime();
+ if (timestamp <= measurementLastTime.get(measurementID)[0]) {
+ // find bad file
+ if (!isBadFile) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ isBadFile = true;
+ badFileNum++;
+ }
+ if (printDetails) {
+ if (timestamp <= measurementLastTime.get(measurementID)[1]) {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[0]) {
+ printBoth(
+ "-------- Timeseries " + measurementID + " overlap between files");
+ hasMeasurementPrintedDetails.get(measurementID)[0] = true;
+ }
+ } else if (timestamp
+ <= lashChunkEndTime.getOrDefault(measurementID, Long.MIN_VALUE)) {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[1]) {
+ printBoth(
+ "-------- Timeseries " + measurementID + " overlap between chunks");
+ hasMeasurementPrintedDetails.get(measurementID)[1] = true;
+ }
+ } else if (timestamp <= lastPageEndTime) {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[2]) {
+ printBoth(
+ "-------- Timeseries " + measurementID + " overlap between pages");
+ hasMeasurementPrintedDetails.get(measurementID)[2] = true;
+ }
+ } else {
+ if (!hasMeasurementPrintedDetails.get(measurementID)[3]) {
+ printBoth(
+ "-------- Timeseries "
+ + measurementID
+ + " overlap within one page");
+ hasMeasurementPrintedDetails.get(measurementID)[3] = true;
+ }
+ }
+ }
+ } else {
+ measurementLastTime.get(measurementID)[0] = timestamp;
+ currentPageEndTime = timestamp;
+ currentChunkEndTime = timestamp;
+ }
+ batchData.next();
+ }
+ dataSize -= pageHeader.getSerializedPageSize();
+ lastPageEndTime = Math.max(lastPageEndTime, currentPageEndTime);
+ }
+ lashChunkEndTime.put(
+ measurementID,
+ Math.max(
+ lashChunkEndTime.getOrDefault(measurementID, Long.MIN_VALUE),
+ currentChunkEndTime));
+ break;
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ if (!deviceID.equals("")) {
+ // record the end time of last device in current file
+ if (resource.getEndTime(deviceID)
+ > deviceEndTime.getOrDefault(deviceID, Long.MIN_VALUE)) {
+ deviceEndTime.put(deviceID, resource.getEndTime(deviceID));
+ }
+ }
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
+ deviceID = chunkGroupHeader.getDeviceID();
+ if (!hasCheckedDeviceOverlap.getOrDefault(deviceID, false)
+ && resource.getStartTime(deviceID)
+ <= deviceEndTime.getOrDefault(deviceID, Long.MIN_VALUE)) {
+ // find bad file
+ if (!isBadFile) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ isBadFile = true;
+ badFileNum++;
+ }
+ if (printDetails) {
+ printBoth("---- Device " + deviceID + " overlap between files");
+ }
+ }
+ hasCheckedDeviceOverlap.put(deviceID, true);
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ reader.readPlanIndex();
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+
+ // record the end time of each timeseries in current file
+ for (Map.Entry<String, Long> entry : lashChunkEndTime.entrySet()) {
+ Long endTime = Math.max(measurementLastTime.get(entry.getKey())[1], entry.getValue());
+ measurementLastTime.get(entry.getKey())[1] = endTime;
+ }
+ }
+ } catch (Throwable e) {
+ logger.error("Meet errors in reading file {} , skip it.", tsFile.getAbsolutePath(), e);
+ printBoth("-- Meet errors in reading file " + tsFile.getAbsolutePath());
+ }
+ }
+ }
+
+ public static boolean checkArgs(String[] args) {
+ if (args.length < 1) {
+ System.out.println(
+ "Please input correct param, which is [path of data dir] [-pd = print details or not] [-f = path of outFile]. Eg: xxx/iotdb/data/data -pd=true -f=xxx/TsFile_validation_view.txt");
+ return false;
+ } else {
+ for (String arg : args) {
+ if (arg.startsWith("-pd")) {
+ printDetails = Boolean.parseBoolean(arg.split("=")[1]);
+ } else if (arg.startsWith("-f")) {
+ printToFile = true;
+ outFilePath = arg.split("=")[1];
+ } else {
+ File f = new File(arg);
+ if (f.isDirectory()
+ && Objects.requireNonNull(
+ f.list(
+ (dir, name) ->
+ (name.equals("sequence") || name.equals("unsequence"))))
+ .length
+ == 2) {
+ File seqDataDir = new File(f, "sequence");
+ seqDataDirList.add(seqDataDir);
+ } else if (arg.endsWith(TSFILE_SUFFIX) && f.isFile()) {
+ fileList.add(f);
+ } else {
+ System.out.println(arg + " is not a correct data directory or tsfile of IOTDB.");
+ return false;
+ }
+ }
+ }
+ if (seqDataDirList.size() == 0 && fileList.size() == 0) {
+ System.out.println(
+ "Please input correct param, which is [path of data dir] [-pd = print details or not] [-f = path of outFile]. Eg: xxx/iotdb/data/data -pd=true -f=xxx/TsFile_validation_view.txt");
+ return false;
+ }
+ return true;
+ }
+ }
+
+ private static boolean checkIsDirectory(File dir) {
+ boolean res = true;
+ if (!dir.isDirectory()) {
+ logger.error("{} is not a directory or does not exist, skip it.", dir.getAbsolutePath());
+ res = false;
+ }
+ return res;
+ }
+
+ private static void printBoth(String msg) {
+ System.out.println(msg);
+ if (printToFile) {
+ pw.println(msg);
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 5a0bca21da..a7ef49bdf9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -73,7 +73,7 @@ public class ChunkMetadata {
// used for ChunkCache, Eg:"root.sg1/0/0"
private String tsFilePrefixPath;
- private ChunkMetadata() {}
+ public ChunkMetadata() {}
/**
* constructor of ChunkMetaData.