You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2019/03/07 02:39:53 UTC
[incubator-iotdb] 01/02: add TsfileChecker
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch east_check_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit ba9a2c96da567326005e6404e4cef373b947ce38
Author: mdf369 <95...@qq.com>
AuthorDate: Thu Feb 28 10:14:50 2019 +0800
add TsfileChecker
---
iotdb/pom.xml | 28 +++
.../bufferwrite/RestorableTsFileIOWriter.java | 8 +
.../org/apache/iotdb/db/utils/TsFileChecker.java | 268 +++++++++++++++++++++
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 28 +--
.../tsfile/file/metadata/ChunkGroupMetaData.java | 6 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 4 +
6 files changed, 325 insertions(+), 17 deletions(-)
diff --git a/iotdb/pom.xml b/iotdb/pom.xml
index 7ff37e9..b8eea29 100644
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@ -134,6 +134,11 @@
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<outputDirectory>${project.basedir}/iotdb/lib</outputDirectory>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.iotdb.db.utils.TsFileChecker</mainClass>
+ </manifest>
+ </archive>
</configuration>
</plugin>
<!--using `mvn test` to run UT, `mvn verify` to run ITs
@@ -163,6 +168,29 @@
<skipITs>${iotdb.it.skip}</skipITs>
</configuration>
</plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4.1</version>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.iotdb.db.utils.TsFileChecker</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<!-- Might require this in Eclipse -->
<!--plugin>
<groupId>org.codehaus.mojo</groupId>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
index 7331104..6e909a9 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
@@ -133,6 +133,14 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter {
}
+ public static int getTsPositionByteSize() {
+ return TS_POSITION_BYTE_SIZE;
+ }
+
+ public static String getRestoreSuffix() {
+ return RESTORE_SUFFIX;
+ }
+
private void recoverMetadata(List<ChunkGroupMetaData> rowGroupMetaDatas) {
// TODO it is better if we can consider the problem caused by deletion
// and re-create time series here.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/TsFileChecker.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/TsFileChecker.java
new file mode 100644
index 0000000..3c78ec4
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/TsFileChecker.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+import org.apache.iotdb.db.engine.bufferwrite.RestorableTsFileIOWriter;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex;
+import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+
+public class TsFileChecker {
+
+ public static int checkTsFile(String path) {
+ TsFileSequenceReader reader = null;
+ long lastPosition = Long.MAX_VALUE;
+ try {
+ boolean complete = true;
+ String restoreFilePath = path + RestorableTsFileIOWriter.getRestoreSuffix();
+ long tsFileLen = new File(path).length();
+ try {
+ lastPosition = readLastPositionFromRestoreFile(restoreFilePath);
+ complete = false;
+ } catch (IOException ex) {
+ lastPosition = tsFileLen;
+ complete = true;
+ }
+
+ if (lastPosition > tsFileLen) {
+ return 0;
+ }
+ if (lastPosition <= TSFileConfig.MAGIC_STRING.length()) {
+ return 2;
+ }
+
+ reader = new TsFileSequenceReader(path, complete);
+ if (!complete) {
+ reader.position(TSFileConfig.MAGIC_STRING.length());
+ }
+ reader.readHeadMagic();
+ byte marker;
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ if (reader.position() > lastPosition) {
+ return 3;
+ }
+
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ ChunkHeader header = reader.readChunkHeader();
+ Decoder defaultTimeDecoder = Decoder.getDecoderByType(
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().timeSeriesEncoder),
+ TSDataType.INT64);
+ Decoder valueDecoder = Decoder
+ .getDecoderByType(header.getEncodingType(), header.getDataType());
+ for (int j = 0; j < header.getNumOfPages(); j++) {
+ PageHeader pageHeader = reader.readPageHeader(header.getDataType());
+ ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
+ PageReader reader1 = new PageReader(pageData, header.getDataType(), valueDecoder,
+ defaultTimeDecoder);
+ while (reader1.hasNextBatch()) {
+ BatchData batchData = reader1.nextBatch();
+ while (batchData.hasNext()) {
+ batchData.currentTime();
+ batchData.currentValue();
+ batchData.next();
+ }
+ }
+ }
+ break;
+ case MetaMarker.CHUNK_GROUP_FOOTER:
+ reader.readChunkGroupFooter();
+ if (reader.position() == lastPosition) {
+ return 2;
+ }
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+
+ TsFileMetaData metaData = reader.readFileMetadata();
+ List<TsDeviceMetadataIndex> deviceMetadataIndexList = metaData.getDeviceMap().values().stream()
+ .sorted((x, y) -> (int) (x.getOffset() - y.getOffset())).collect(Collectors.toList());
+ for (TsDeviceMetadataIndex index : deviceMetadataIndexList) {
+ TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index);
+ deviceMetadata.getChunkGroupMetaDataList();
+ }
+ reader.readTailMagic();
+ reader.close();
+
+ return 1;
+ } catch (Exception ex) {
+// System.out.println(ex);
+ try {
+ if (reader != null || reader.position() > lastPosition) {
+ return 3;
+ } else {
+ return 0;
+ }
+ } catch (IOException e) {
+ return 0;
+ }
+ }
+ }
+
+ private static long readLastPositionFromRestoreFile(String path) throws IOException {
+ int tsfilePositionByteSize = RestorableTsFileIOWriter.getTsPositionByteSize();
+ byte[] lastPostionBytes = new byte[tsfilePositionByteSize];
+ RandomAccessFile randomAccessFile = null;
+ randomAccessFile = new RandomAccessFile(path, "r");
+
+ long fileLength = randomAccessFile.length();
+ randomAccessFile.seek(fileLength - tsfilePositionByteSize);
+ randomAccessFile.read(lastPostionBytes);
+ long lastPosition = BytesUtils.bytesToLong(lastPostionBytes);
+ randomAccessFile.close();
+ return lastPosition;
+ }
+
+ public static List<File> getFileList(String dirPath) {
+ List<File> res = new ArrayList<>();
+ File dir = new File(dirPath);
+ if (dir.isFile()) {
+ res.add(dir);
+ return res;
+ }
+
+ File[] files = dir.listFiles();
+ if (files != null) {
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].isDirectory()) {
+ res.addAll(getFileList(files[i].getAbsolutePath()));
+ } else {
+ res.add(files[i]);
+ }
+ }
+
+ }
+ return res;
+ }
+
+ public static List<File> getLastDirList(String dirPath) {
+ List<File> res = new ArrayList<>();
+ File dir = new File(dirPath);
+ if (dir.isFile()) {
+ return Collections.emptyList();
+ }
+
+ File[] files = dir.listFiles();
+ if (files != null) {
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].isDirectory()) {
+ res.addAll(getLastDirList(files[i].getAbsolutePath()));
+ } else {
+ res.add(dir);
+ return res;
+ }
+ }
+
+ }
+ return res;
+ }
+
+ public static int getRestoreFileNum(File dir) throws Exception {
+ int count = 0;
+ if (dir.isFile()) {
+ throw new Exception("Path: " + dir.getAbsolutePath() + " is a file!");
+ }
+
+ File[] files = dir.listFiles();
+ if (files != null) {
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].isFile() && files[i].getPath().endsWith(RestorableTsFileIOWriter.getRestoreSuffix())) {
+ count++;
+ }
+ }
+
+ }
+ return count;
+ }
+
+ public static void main(String[] args) throws Exception {
+// args = new String[]{"/Users/East/Desktop/tsfile"};
+
+ if (args == null || args.length < 1) {
+ System.out.println("Please input root dir!");
+ System.exit(1);
+ }
+
+ String root = args[0];
+
+ List<File> lastDirList = TsFileChecker.getLastDirList(root);
+ List<String> wrongRestoreNumDirList = new ArrayList<>();
+ for (int i = 0; i < lastDirList.size(); i++) {
+ int count = TsFileChecker.getRestoreFileNum(lastDirList.get(i));
+ if (count > 1) {
+ wrongRestoreNumDirList.add(lastDirList.get(i).getAbsolutePath());
+ }
+ }
+ System.out.println("Directories of more than one restore file:");
+ for (int i = 0; i < wrongRestoreNumDirList.size(); i++) {
+ System.out.println(wrongRestoreNumDirList.get(i));
+ }
+ System.out.println();
+
+ List<File> fileList = TsFileChecker.getFileList(root);
+ Map<Integer, List<String>> tsfileStatusMap = new HashMap<>();
+ tsfileStatusMap.put(0, new ArrayList<>());
+ tsfileStatusMap.put(1, new ArrayList<>());
+ tsfileStatusMap.put(2, new ArrayList<>());
+ tsfileStatusMap.put(3, new ArrayList<>());
+ int num = fileList.size();
+ System.out.println("Num of file: " + num);
+ for (int i = 0; i < fileList.size(); i++) {
+ System.out.println("Check No." + (i + 1));
+ String filePath = fileList.get(i).getAbsolutePath();
+ if (filePath.endsWith(RestorableTsFileIOWriter.getRestoreSuffix())) {
+ num--;
+ continue;
+ }
+ int status = TsFileChecker.checkTsFile(filePath);
+ tsfileStatusMap.get(status).add(filePath);
+ }
+ System.out.println("Num of TsFile: " + num);
+ System.out.println("TsFile status:");
+ for (Entry<Integer, List<String>> integerListEntry : tsfileStatusMap.entrySet()) {
+ System.out.println(integerListEntry.getKey() + ": " + integerListEntry.getValue().size());
+// integerListEntry.getValue().forEach(value -> System.out.println(value));
+ }
+ }
+}
diff --git a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
index f5caeaa..449f953 100644
--- a/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
+++ b/tsfile/example/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
@@ -43,7 +43,7 @@ import org.apache.iotdb.tsfile.read.reader.page.PageReader;
public class TsFileSequenceRead {
public static void main(String[] args) throws IOException {
- TsFileSequenceReader reader = new TsFileSequenceReader("test.tsfile");
+ TsFileSequenceReader reader = new TsFileSequenceReader("/Users/East/Desktop/tsfile/12272464235000-1550256931390");
System.out.println("file length: " + new File("test.tsfile").length());
System.out.println("file magic head: " + reader.readHeadMagic());
System.out.println("file magic tail: " + reader.readTailMagic());
@@ -75,19 +75,19 @@ public class TsFileSequenceRead {
System.out.println("\t\tPage data position: " + reader.position());
System.out.println("\t\tpoints in the page: " + pageHeader.getNumOfValues());
ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
- System.out
- .println("\t\tUncompressed page data size: " + pageHeader.getUncompressedSize());
- PageReader reader1 = new PageReader(pageData, header.getDataType(), valueDecoder,
- defaultTimeDecoder);
- while (reader1.hasNextBatch()) {
- BatchData batchData = reader1.nextBatch();
- while (batchData.hasNext()) {
- System.out.println(
- "\t\t\ttime, value: " + batchData.currentTime() + ", " + batchData
- .currentValue());
- batchData.next();
- }
- }
+// System.out
+// .println("\t\tUncompressed page data size: " + pageHeader.getUncompressedSize());
+// PageReader reader1 = new PageReader(pageData, header.getDataType(), valueDecoder,
+// defaultTimeDecoder);
+// while (reader1.hasNextBatch()) {
+// BatchData batchData = reader1.nextBatch();
+// while (batchData.hasNext()) {
+// System.out.println(
+// "\t\t\ttime, value: " + batchData.currentTime() + ", " + batchData
+// .currentValue());
+// batchData.next();
+// }
+// }
}
break;
case MetaMarker.CHUNK_GROUP_FOOTER:
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
index 01cc0ab..b0a8774 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java
@@ -127,9 +127,9 @@ public class ChunkGroupMetaData {
ChunkGroupMetaData chunkGroupMetaData = new ChunkGroupMetaData();
chunkGroupMetaData.deviceID = ReadWriteIOUtils.readString(buffer);
- chunkGroupMetaData.startOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer);
- chunkGroupMetaData.endOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer);
- chunkGroupMetaData.version = ReadWriteIOUtils.readLong(buffer);
+// chunkGroupMetaData.startOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer);
+// chunkGroupMetaData.endOffsetOfChunkGroup = ReadWriteIOUtils.readLong(buffer);
+// chunkGroupMetaData.version = ReadWriteIOUtils.readLong(buffer);
int size = ReadWriteIOUtils.readInt(buffer);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index fa673ba..19f781d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -267,6 +267,10 @@ public class TsFileSequenceReader {
return tsFileInput.position();
}
+ public void position(long position) throws IOException {
+ tsFileInput.position(position);
+ }
+
public ByteBuffer readPage(PageHeader header, CompressionType type) throws IOException {
return readPage(header, type, -1);
}