You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/04/22 02:19:20 UTC
[iotdb] branch master updated: [IOTDB-2835]Fix empty page in selfcheck method of TsFileSequenceReader (#5552)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9522cfb39e [IOTDB-2835]Fix empty page in selfcheck method of TsFileSequenceReader (#5552)
9522cfb39e is described below
commit 9522cfb39e33d3cb62723bc36f431f35c6c26cfb
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Fri Apr 22 10:19:15 2022 +0800
[IOTDB-2835]Fix empty page in selfcheck method of TsFileSequenceReader (#5552)
---
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 8 +++-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 6 ++-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 18 ++++++---
.../apache/iotdb/tsfile/write/TsFileWriter.java | 4 --
.../tsfile/read/TsFileSequenceReaderTest.java | 47 ++++++++++++++++++++++
5 files changed, 72 insertions(+), 11 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 62af97b895..22aa5d1996 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -175,6 +175,7 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -1418,7 +1419,12 @@ public class PlanExecutor implements IPlanExecutor {
List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) {
- reader.selfCheck(schemaMap, chunkGroupMetadataList, false);
+ if (reader.selfCheck(schemaMap, chunkGroupMetadataList, false)
+ != TsFileCheckStatus.COMPLETE_FILE) {
+ throw new QueryProcessException(
+ String.format(
+ "Cannot load file %s because the file has crashed.", file.getAbsolutePath()));
+ }
if (plan.getVerifyMetadata()) {
loadNewTsFileVerifyMetadata(reader);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index 24e2226209..0146db0ef3 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -85,7 +86,10 @@ public class TsFileSketchTool {
// get metadata information
tsFileMetaData = reader.readFileMetadata();
allChunkGroupMetadata = new ArrayList<>();
- reader.selfCheck(null, allChunkGroupMetadata, false);
+ if (reader.selfCheck(null, allChunkGroupMetadata, false) != TsFileCheckStatus.COMPLETE_FILE) {
+ throw new IOException(
+ String.format("Cannot load file %s because the file has crashed.", filename));
+ }
} catch (IOException e) {
e.printStackTrace();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 88834f121b..584d398215 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1257,15 +1257,16 @@ public class TsFileSequenceReader implements AutoCloseable {
}
tsFileInput.position(headerLength);
+ boolean isComplete = isComplete();
if (fileSize == headerLength) {
return headerLength;
- } else if (isComplete()) {
+ } else if (isComplete) {
loadMetadataSize();
if (fastFinish) {
return TsFileCheckStatus.COMPLETE_FILE;
}
}
- // not a complete file, we will recover it...
+ // if not a complete file, we will recover it...
long truncatedSize = headerLength;
byte marker;
List<long[]> timeBatch = new ArrayList<>();
@@ -1309,7 +1310,10 @@ public class TsFileSequenceReader implements AutoCloseable {
while (dataSize > 0) {
// a new Page
PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), true);
- chunkStatistics.mergeStatistics(pageHeader.getStatistics());
+ if (pageHeader.getUncompressedSize() != 0) {
+ // not empty page
+ chunkStatistics.mergeStatistics(pageHeader.getStatistics());
+ }
this.skipPageData(pageHeader);
dataSize -= pageHeader.getSerializedPageSize();
chunkHeader.increasePageNums(1);
@@ -1476,9 +1480,13 @@ public class TsFileSequenceReader implements AutoCloseable {
// last chunk group Metadata
chunkGroupMetadataList.add(new ChunkGroupMetadata(lastDeviceId, chunkMetadataList));
}
- truncatedSize = this.position() - 1;
+ if (isComplete) {
+ truncatedSize = TsFileCheckStatus.COMPLETE_FILE;
+ } else {
+ truncatedSize = this.position() - 1;
+ }
} catch (Exception e) {
- logger.info(
+ logger.warn(
"TsFile {} self-check cannot proceed at position {} " + "recovered, because : {}",
file,
this.position(),
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
index f2ced038c9..ec2ef2baf5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
@@ -636,8 +636,4 @@ public class TsFileWriter implements AutoCloseable {
public TsFileIOWriter getIOWriter() {
return this.fileWriter;
}
-
- public void setIsUnseq(boolean unseq) {
- this.isUnseq = unseq;
- }
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
index d54e4be7e0..a69eec5bfb 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java
@@ -20,19 +20,28 @@
package org.apache.iotdb.tsfile.read;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.FileGenerator;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsFileGeneratorUtils;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -125,4 +134,42 @@ public class TsFileSequenceReaderTest {
Assert.assertTrue(reader.readChunkMetadataInDevice("d3").isEmpty());
reader.close();
}
+
+ @Test
+ public void testReadEmptyPageInSelfCheck() throws IOException, WriteProcessException {
+ int oldMaxPagePointNum =
+ TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10);
+ File testFile = new File(FILE_PATH);
+
+ // create tsfile with empty page
+ try (TsFileWriter tsFileWriter = new TsFileWriter(testFile)) {
+ // register aligned timeseries
+ List<MeasurementSchema> alignedMeasurementSchemas = new ArrayList<>();
+ alignedMeasurementSchemas.add(
+ new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN));
+ alignedMeasurementSchemas.add(
+ new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN));
+ tsFileWriter.registerAlignedTimeseries(new Path("d1"), alignedMeasurementSchemas);
+
+ List<MeasurementSchema> writeMeasurementScheams = new ArrayList<>();
+ // only write s1
+ writeMeasurementScheams.add(alignedMeasurementSchemas.get(0));
+ TsFileGeneratorUtils.writeWithTsRecord(
+ tsFileWriter, "d1", writeMeasurementScheams, 25, 0, 0, true);
+
+ // write s1 and s2, fill 2 empty pages for s2
+ writeMeasurementScheams.add(alignedMeasurementSchemas.get(1));
+ TsFileGeneratorUtils.writeWithTsRecord(
+ tsFileWriter, "d1", writeMeasurementScheams, 10, 25, 0, true);
+ } finally {
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldMaxPagePointNum);
+ }
+
+ // read tsfile with selfCheck method
+ TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH);
+ Assert.assertEquals(
+ TsFileCheckStatus.COMPLETE_FILE,
+ reader.selfCheck(new HashMap<>(), new ArrayList<>(), false));
+ }
}