You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/09/30 09:45:03 UTC
(tsfile) branch develop updated: fix(cpp): stop misreading pages after an empty value page (#976)
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 05b5d939c fix(cpp): stop misreading pages after an empty value page (#976)
05b5d939c is described below
commit 05b5d939c5e69f3b9ad83afdfbb200d6bc50df42
Author: Colin Lee <10...@users.noreply.github.com>
AuthorDate: Wed Sep 30 17:44:57 2026 +0800
fix(cpp): stop misreading pages after an empty value page (#976)
* fix(cpp): read empty value page headers without consuming the next page (#971)
Java writes an all-null value page as a single zero varint and omits the
compressed size and statistics (ValueChunkWriter.writeEmptyPageToPageBuffer).
The C++ PageHeader::deserialize_from always read a second varint and, when
requested, a statistic, so after an empty page it consumed the following
page's bytes as the empty page's header and misread every later page.
Return early when uncompressed_size_ == 0, matching the Java format. The
aligned readers already treat compressed_size_ == 0 as an all-null page, so
the fix is localized to PageHeader deserialization.
Add PageHeader unit tests covering an empty page followed by a non-empty
page, for both deserialize_stat values, asserting it consumes exactly one
varint and leaves the following page intact.
* test: cover empty pages in cross-language fixtures
---
cpp/src/common/tsfile_common.h | 6 +
cpp/test/common/tsfile_common_test.cc | 41 ++++
...odel_encoding_compression_compatibility_test.cc | 149 +++++++++++---
...eModelEncodingCompressionCompatibilityTest.java | 218 ++++++++++++++++++---
4 files changed, 369 insertions(+), 45 deletions(-)
diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h
index ee3b389d5..bfe597d8a 100644
--- a/cpp/src/common/tsfile_common.h
+++ b/cpp/src/common/tsfile_common.h
@@ -78,6 +78,12 @@ struct PageHeader {
int ret = common::E_OK;
if (RET_FAIL(common::SerializationUtil::read_var_uint(
uncompressed_size_, in))) {
+ } else if (uncompressed_size_ == 0) {
+ // Empty page: the Java writer stores only a single zero varint and
+ // writes no compressed size or statistics (see
+ // ValueChunkWriter.writeEmptyPageToPageBuffer). Reading further
+ // would consume the following page's bytes as this page's header.
+ compressed_size_ = 0;
} else if (RET_FAIL(common::SerializationUtil::read_var_uint(
compressed_size_, in))) {
} else if (deserialize_stat) {
diff --git a/cpp/test/common/tsfile_common_test.cc b/cpp/test/common/tsfile_common_test.cc
index 309fdbc39..2f28a0b74 100644
--- a/cpp/test/common/tsfile_common_test.cc
+++ b/cpp/test/common/tsfile_common_test.cc
@@ -43,6 +43,47 @@ TEST(PageHeaderTest, Reset) {
EXPECT_EQ(header.compressed_size_, 0);
}
+TEST(PageHeaderTest, DeserializeEmptyPageConsumesSingleVarint) {
+ // Java writes an empty value page as a single zero varint (see
+ // ValueChunkWriter.writeEmptyPageToPageBuffer): it carries no compressed
+ // size and no statistics. The C++ deserializer must not read past it.
+ common::ByteStream in(1024, common::MOD_DEFAULT);
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, in));
+ // A following non-empty page header (uncompressed=10, compressed=5).
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(10, in));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(5, in));
+
+ PageHeader header;
+ ASSERT_EQ(common::E_OK,
+ header.deserialize_from(in, false, common::TSDataType::INT32));
+ EXPECT_EQ(header.uncompressed_size_, 0U);
+ EXPECT_EQ(header.compressed_size_, 0U);
+ EXPECT_EQ(header.statistic_, nullptr);
+ // The empty page header consumes exactly one byte (a single varint),
+ // leaving the following page header intact.
+ EXPECT_EQ(1U, in.read_pos());
+
+ PageHeader next;
+ ASSERT_EQ(common::E_OK,
+ next.deserialize_from(in, false, common::TSDataType::INT32));
+ EXPECT_EQ(next.uncompressed_size_, 10U);
+ EXPECT_EQ(next.compressed_size_, 5U);
+ EXPECT_EQ(next.statistic_, nullptr);
+}
+
+TEST(PageHeaderTest, DeserializeEmptyPageWithStatisticConsumesSingleVarint) {
+ common::ByteStream in(1024, common::MOD_DEFAULT);
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, in));
+
+ PageHeader header;
+ ASSERT_EQ(common::E_OK,
+ header.deserialize_from(in, true, common::TSDataType::INT32));
+ EXPECT_EQ(header.uncompressed_size_, 0U);
+ EXPECT_EQ(header.compressed_size_, 0U);
+ EXPECT_EQ(header.statistic_, nullptr);
+ EXPECT_EQ(1U, in.read_pos());
+}
+
TEST(ChunkHeaderTest, DefaultConstructor) {
ChunkHeader header;
EXPECT_EQ(header.measurement_name_, "");
diff --git a/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc b/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc
index 09ed3e49b..8dedc34eb 100644
--- a/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc
+++ b/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc
@@ -28,6 +28,7 @@
#include <cstring>
#include <fstream>
#include <limits>
+#include <memory>
#include <sstream>
#include <string>
#include <unordered_set>
@@ -43,6 +44,7 @@
#endif
#include <fcntl.h>
+#include "common/config/config.h"
#include "common/db_common.h"
#include "common/schema.h"
#include "common/tablet.h"
@@ -62,11 +64,14 @@ const char* const kGenerateDirEnv = "TSFILE_COMPAT_GENERATE_DIR";
const char* const kValidateDirEnv = "TSFILE_COMPAT_VALIDATE_DIR";
const char* const kManifestFile = "manifest.csv";
const char* const kManifestHeader =
- "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount";
+ "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount,"
+ "nullPattern";
const char* const kTableName = "compat_table";
const char* const kTagColumn = "device";
const char* const kValueColumn = "value";
const char* const kTagValue = "compat_device";
+const char* const kAnchorColumn = "anchor";
+const int kNullPageRows = 64;
// Crosses the 129-value TS_2DIFF block boundary (300 -> 129+129+42), so
// page-wide bitmaps must survive block transitions. Applied to every case.
constexpr int kRowCount = 300;
@@ -181,6 +186,7 @@ struct FixtureCase {
TSEncoding encoding;
CompressionType compression;
int row_count;
+ std::string null_pattern = "NONE";
FixtureCase()
: table_name(kTableName),
@@ -221,7 +227,8 @@ struct FixtureCase {
out << file_name << "," << table_name << "," << tag_column << ","
<< value_column << "," << get_data_type_name(data_type) << ","
<< get_encoding_name(encoding) << ","
- << get_compression_name(compression) << "," << row_count;
+ << get_compression_name(compression) << "," << row_count << ","
+ << null_pattern;
return out.str();
}
};
@@ -264,7 +271,7 @@ bool ParseCompressionName(const std::string& value, CompressionType& out) {
FixtureCase ParseManifestLine(const std::string& line) {
FixtureCase fixture_case;
std::vector<std::string> parts = Split(line, ',');
- if (parts.size() != 8) {
+ if (parts.size() != 9) {
ADD_FAILURE() << "Bad manifest line: " << line;
return fixture_case;
}
@@ -282,6 +289,10 @@ FixtureCase ParseManifestLine(const std::string& line) {
ADD_FAILURE() << "Bad compression in manifest line: " << line;
}
fixture_case.row_count = std::atoi(parts[7].c_str());
+ fixture_case.null_pattern = parts[8];
+ EXPECT_TRUE(fixture_case.null_pattern == "NONE" ||
+ fixture_case.null_pattern == "MIXED" ||
+ fixture_case.null_pattern == "ALL");
return fixture_case;
}
@@ -487,9 +498,55 @@ std::vector<FixtureCase> BuildMatrix() {
}
cases.emplace_back(DOUBLE, CAMEL, compression, kRowCount);
}
+ for (CompressionType compression : {UNCOMPRESSED, LZ4, ZSTD, LZMA2}) {
+ for (TSDataType data_type : data_types) {
+ for (const char* pattern : {"MIXED", "ALL"}) {
+ FixtureCase fixture(data_type, PLAIN, compression,
+ kNullPageRows * 8);
+ fixture.null_pattern = pattern;
+ fixture.file_name += "." + FixtureCase::Lower(pattern) + ".512";
+ cases.push_back(fixture);
+ }
+ FixtureCase fixture(data_type, PLAIN, compression, kNullPageRows);
+ fixture.null_pattern = "ALL";
+ fixture.file_name += ".all.64";
+ cases.push_back(fixture);
+ }
+ }
return cases;
}
+// Shared contract with the Java test: empty first/middle/consecutive/trailing
+// pages, non-empty pages 1/5, and alternating null values inside page 4.
+bool IsNull(const FixtureCase& fixture_case, int row) {
+ if (fixture_case.null_pattern == "ALL") return true;
+ if (fixture_case.null_pattern == "NONE") return false;
+ int page = row / kNullPageRows;
+ return page == 0 || page == 2 || page == 3 || page >= 6 ||
+ (page == 4 && row % 2 == 0);
+}
+
+// Restore process-wide settings even when a fatal assertion returns early.
+class ScopedNullPageSize {
+ public:
+ explicit ScopedNullPageSize(const FixtureCase& fixture_case)
+ : old_rows_(g_config_value_.page_writer_max_point_num_),
+ old_bytes_(g_config_value_.page_writer_max_memory_bytes_) {
+ if (fixture_case.null_pattern != "NONE") {
+ g_config_value_.page_writer_max_point_num_ = kNullPageRows;
+ g_config_value_.page_writer_max_memory_bytes_ = 1024 * 1024;
+ }
+ }
+ ~ScopedNullPageSize() {
+ g_config_value_.page_writer_max_point_num_ = old_rows_;
+ g_config_value_.page_writer_max_memory_bytes_ = old_bytes_;
+ }
+
+ private:
+ uint32_t old_rows_;
+ uint32_t old_bytes_;
+};
+
TableSchema* CreateTableSchema(const FixtureCase& fixture_case) {
std::vector<MeasurementSchema*> measurement_schemas;
std::vector<ColumnCategory> column_categories;
@@ -500,6 +557,11 @@ TableSchema* CreateTableSchema(const FixtureCase& fixture_case) {
new MeasurementSchema(fixture_case.value_column, fixture_case.data_type,
fixture_case.encoding, fixture_case.compression));
column_categories.emplace_back(ColumnCategory::FIELD);
+ if (fixture_case.null_pattern != "NONE") {
+ measurement_schemas.emplace_back(new MeasurementSchema(
+ kAnchorColumn, INT64, PLAIN, fixture_case.compression));
+ column_categories.emplace_back(ColumnCategory::FIELD);
+ }
return new TableSchema(fixture_case.table_name, measurement_schemas,
column_categories);
}
@@ -543,13 +605,21 @@ Tablet CreateTablet(TableSchema* table_schema,
for (int row = 0; row < fixture_case.row_count; ++row) {
EXPECT_EQ(E_OK, tablet.add_timestamp(row, row));
EXPECT_EQ(E_OK, tablet.add_value(row, kTagColumn, kTagValue));
- AddValue(tablet, fixture_case, row);
+ // Tablet entries remain null until add_value is called.
+ if (!IsNull(fixture_case, row)) {
+ AddValue(tablet, fixture_case, row);
+ }
+ if (fixture_case.null_pattern != "NONE") {
+ EXPECT_EQ(E_OK, tablet.add_value(row, kAnchorColumn,
+ static_cast<int64_t>(row)));
+ }
}
return tablet;
}
void WriteFixture(const std::string& directory,
const FixtureCase& fixture_case) {
+ ScopedNullPageSize page_size(fixture_case);
std::string file_name = JoinPath(directory, fixture_case.file_name);
remove(file_name.c_str());
WriteFile write_file;
@@ -678,21 +748,30 @@ void AssertOnWireCodec(const std::string& directory,
}
}
-void ValidateFixture(const std::string& directory,
- const FixtureCase& fixture_case) {
- SCOPED_TRACE(fixture_case.ToManifestLine());
- AssertOnWireCodec(directory, fixture_case);
+void ValidateQuery(const std::string& directory,
+ const FixtureCase& fixture_case, bool with_anchor, int start,
+ int end) {
+ SCOPED_TRACE(::testing::Message()
+ << fixture_case.ToManifestLine() << " anchor=" << with_anchor
+ << " range=" << start << ":" << end);
TsFileReader reader;
ASSERT_EQ(E_OK, reader.open(JoinPath(directory, fixture_case.file_name)));
+ std::vector<std::string> columns = {fixture_case.tag_column,
+ fixture_case.value_column};
+ if (with_anchor) columns.push_back(kAnchorColumn);
ResultSet* raw_result_set = nullptr;
- ASSERT_EQ(E_OK,
- reader.query(fixture_case.table_name,
- {fixture_case.tag_column, fixture_case.value_column},
- INT64_MIN, INT64_MAX, raw_result_set));
+ ASSERT_EQ(E_OK, reader.query(fixture_case.table_name, columns, start, end,
+ raw_result_set));
ASSERT_NE(nullptr, raw_result_set);
auto* result_set = static_cast<TableResultSet*>(raw_result_set);
+ // Assertion failures must release the query before its reader is destroyed.
+ auto cleanup = [&reader](TableResultSet* result) {
+ reader.destroy_query_data_set(result);
+ };
+ std::unique_ptr<TableResultSet, decltype(cleanup)> owned_result(result_set,
+ cleanup);
std::shared_ptr<ResultSetMetadata> metadata = result_set->get_metadata();
- ASSERT_EQ(3U, metadata->get_column_count());
+ ASSERT_EQ(with_anchor ? 4U : 3U, metadata->get_column_count());
ASSERT_EQ("time", metadata->get_column_name(1));
ASSERT_EQ(INT64, metadata->get_column_type(1));
ASSERT_EQ(fixture_case.tag_column, metadata->get_column_name(2));
@@ -701,27 +780,51 @@ void ValidateFixture(const std::string& directory,
ASSERT_EQ(fixture_case.data_type, metadata->get_column_type(3));
bool has_next = false;
- int row = 0;
+ int row = start;
while (true) {
ASSERT_EQ(E_OK, result_set->next(has_next));
- if (!has_next) {
- break;
- }
+ if (!has_next) break;
SCOPED_TRACE(::testing::Message() << "row=" << row);
+ ASSERT_LE(row, end);
ASSERT_EQ(row, result_set->get_value<int64_t>(1));
ASSERT_FALSE(result_set->is_null(2));
ASSERT_EQ(kTagValue,
result_set->get_value<common::String*>(2)->to_std_string());
- ASSERT_FALSE(result_set->is_null(3));
- AssertValue(fixture_case, row, result_set);
+ ASSERT_EQ(IsNull(fixture_case, row), result_set->is_null(3));
+ if (!IsNull(fixture_case, row)) {
+ ASSERT_NO_FATAL_FAILURE(AssertValue(fixture_case, row, result_set));
+ }
+ if (with_anchor) {
+ ASSERT_FALSE(result_set->is_null(4));
+ ASSERT_EQ(row, result_set->get_value<int64_t>(4));
+ }
++row;
}
- ASSERT_EQ(fixture_case.row_count, row);
- result_set->close();
- reader.destroy_query_data_set(result_set);
+ ASSERT_EQ(end + 1, row);
+ owned_result.reset();
ASSERT_EQ(E_OK, reader.close());
}
+void ValidateFixture(const std::string& directory,
+ const FixtureCase& fixture_case) {
+ SCOPED_TRACE(fixture_case.ToManifestLine());
+ ASSERT_NO_FATAL_FAILURE(AssertOnWireCodec(directory, fixture_case));
+ ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, false, 0,
+ fixture_case.row_count - 1));
+ if (fixture_case.null_pattern != "NONE") {
+ ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, true, 0,
+ fixture_case.row_count - 1));
+ if (fixture_case.row_count > kNullPageRows) {
+ ASSERT_NO_FATAL_FAILURE(
+ ValidateQuery(directory, fixture_case, false, 63, 321));
+ ASSERT_NO_FATAL_FAILURE(
+ ValidateQuery(directory, fixture_case, true, 128, 255));
+ ASSERT_NO_FATAL_FAILURE(
+ ValidateQuery(directory, fixture_case, true, 320, 383));
+ }
+ }
+}
+
} // namespace
class TableModelEncodingCompressionCompatibilityTest : public ::testing::Test {
@@ -751,6 +854,6 @@ TEST_F(TableModelEncodingCompressionCompatibilityTest, ValidateFixtures) {
}
std::vector<FixtureCase> cases = ReadManifest(input_dir);
for (const FixtureCase& fixture_case : cases) {
- ValidateFixture(input_dir, fixture_case);
+ ASSERT_NO_FATAL_FAILURE(ValidateFixture(input_dir, fixture_case));
}
}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java
index 80e67bc7f..033aea3a9 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java
@@ -19,12 +19,18 @@
package org.apache.tsfile.compatibility;
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encoding.encoder.Encoder;
import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Chunk;
import org.apache.tsfile.read.query.dataset.ResultSet;
import org.apache.tsfile.read.query.dataset.ResultSetMetadata;
import org.apache.tsfile.read.v4.ITsFileReader;
@@ -40,6 +46,7 @@ import org.junit.Test;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -54,6 +61,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
public class TableModelEncodingCompressionCompatibilityTest {
@@ -61,7 +69,7 @@ public class TableModelEncodingCompressionCompatibilityTest {
private static final String VALIDATE_DIR_PROPERTY = "tsfile.compat.validate.dir";
private static final String MANIFEST_FILE = "manifest.csv";
private static final String MANIFEST_HEADER =
- "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount";
+ "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount,nullPattern";
private static final String TABLE_NAME = "compat_table";
private static final String TAG_COLUMN = "device";
private static final String VALUE_COLUMN = "value";
@@ -69,6 +77,28 @@ public class TableModelEncodingCompressionCompatibilityTest {
// Crosses the 129-value TS_2DIFF block boundary (300 -> 129+129+42), so
// page-wide bitmaps must survive block transitions. Applied to every case.
private static final int ROW_COUNT = 300;
+ private static final int NULL_PAGE_ROWS = 64;
+ private static final String ANCHOR_COLUMN = "anchor";
+
+ // Keep this contract in sync with the C++ compatibility test. MIXED has
+ // empty pages at the start, in the middle (consecutive), and at the end;
+ // page 4 mixes null/non-null rows and pages 1/5 are entirely non-null.
+ private enum NullPattern {
+ NONE,
+ MIXED,
+ ALL
+ }
+
+ private static boolean isNull(FixtureCase fixtureCase, int row) {
+ if (fixtureCase.nullPattern == NullPattern.ALL) {
+ return true;
+ }
+ if (fixtureCase.nullPattern == NullPattern.NONE) {
+ return false;
+ }
+ int page = row / NULL_PAGE_ROWS;
+ return page == 0 || page == 2 || page == 3 || page >= 6 || (page == 4 && row % 2 == 0);
+ }
private static final int[] INT_VALUES = {
0,
@@ -260,28 +290,104 @@ public class TableModelEncodingCompressionCompatibilityTest {
}
cases.add(new FixtureCase(TSDataType.DOUBLE, TSEncoding.CAMEL, compression, ROW_COUNT));
}
+ for (CompressionType compression :
+ Arrays.asList(
+ CompressionType.UNCOMPRESSED,
+ CompressionType.LZ4,
+ CompressionType.ZSTD,
+ CompressionType.LZMA2)) {
+ for (TSDataType dataType :
+ Arrays.asList(
+ TSDataType.INT32,
+ TSDataType.DATE,
+ TSDataType.INT64,
+ TSDataType.TIMESTAMP,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE)) {
+ for (NullPattern pattern : Arrays.asList(NullPattern.MIXED, NullPattern.ALL)) {
+ cases.add(new FixtureCase(dataType, compression, NULL_PAGE_ROWS * 8, pattern));
+ }
+ cases.add(new FixtureCase(dataType, compression, NULL_PAGE_ROWS, NullPattern.ALL));
+ }
+ }
return cases;
}
private static void writeFixture(Path directory, FixtureCase fixtureCase) throws Exception {
TableSchema tableSchema = tableSchema(fixtureCase);
File file = directory.resolve(fixtureCase.fileName).toFile();
- try (TsFileWriter writer = new TsFileWriter(file)) {
- writer.setGenerateTableSchema(true);
- writer.registerTableSchema(tableSchema);
- writer.writeTable(tablet(tableSchema, fixtureCase));
+ TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
+ int oldPageRows = config.getMaxNumberOfPointsInPage();
+ int oldPageBytes = config.getPageSizeInByte();
+ try {
+ if (fixtureCase.nullPattern != NullPattern.NONE) {
+ config.setMaxNumberOfPointsInPage(NULL_PAGE_ROWS);
+ config.setPageSizeInByte(1024 * 1024);
+ }
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ writer.setGenerateTableSchema(true);
+ writer.registerTableSchema(tableSchema);
+ writer.writeTable(tablet(tableSchema, fixtureCase));
+ }
+ } finally {
+ config.setMaxNumberOfPointsInPage(oldPageRows);
+ config.setPageSizeInByte(oldPageBytes);
+ }
+ if (fixtureCase.nullPattern == NullPattern.MIXED) {
+ assertEmptyPages(file, fixtureCase);
+ }
+ }
+
+ // Prove that the regression fixture actually contains Java's one-byte empty
+ // page headers, rather than merely null values inside a non-empty page.
+ private static void assertEmptyPages(File file, FixtureCase fixtureCase) throws IOException {
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) {
+ List<ChunkMetadata> chunks =
+ reader.getChunkMetadataList(
+ reader.getAllDevices().get(0), fixtureCase.valueColumn, false);
+ assertEquals(1, chunks.size());
+ Chunk chunk = reader.readMemChunk(chunks.get(0));
+ ByteBuffer data = chunk.getData();
+ int page = 0;
+ while (data.hasRemaining()) {
+ int start = data.position();
+ PageHeader header = PageHeader.deserializeFrom(data, fixtureCase.dataType);
+ boolean empty = page == 0 || page == 2 || page == 3 || page == 6 || page == 7;
+ assertEquals("empty page " + page, empty, header.getUncompressedSize() == 0);
+ if (empty) {
+ assertEquals(1, data.position() - start);
+ }
+ data.position(data.position() + header.getCompressedSize());
+ page++;
+ }
+ assertEquals(8, page);
}
}
private static void validateFixture(Path directory, FixtureCase fixtureCase) throws Exception {
+ validateQuery(directory, fixtureCase, false, 0, fixtureCase.rowCount - 1);
+ if (fixtureCase.nullPattern != NullPattern.NONE) {
+ validateQuery(directory, fixtureCase, true, 0, fixtureCase.rowCount - 1);
+ if (fixtureCase.rowCount > NULL_PAGE_ROWS) {
+ // Start inside/at page boundaries and query an entirely null interval.
+ validateQuery(directory, fixtureCase, false, 63, 321);
+ validateQuery(directory, fixtureCase, true, 128, 255);
+ validateQuery(directory, fixtureCase, true, 320, 383);
+ }
+ }
+ }
+
+ private static void validateQuery(
+ Path directory, FixtureCase fixtureCase, boolean withAnchor, int start, int end)
+ throws Exception {
File file = directory.resolve(fixtureCase.fileName).toFile();
+ List<String> columns =
+ new ArrayList<>(Arrays.asList(fixtureCase.tagColumn, fixtureCase.valueColumn));
+ if (withAnchor) {
+ columns.add(ANCHOR_COLUMN);
+ }
try (ITsFileReader reader = new TsFileReaderBuilder().file(file).build();
- ResultSet resultSet =
- reader.query(
- fixtureCase.tableName,
- Arrays.asList(fixtureCase.tagColumn, fixtureCase.valueColumn),
- Long.MIN_VALUE,
- Long.MAX_VALUE)) {
+ ResultSet resultSet = reader.query(fixtureCase.tableName, columns, start, end)) {
ResultSetMetadata metadata = resultSet.getMetadata();
assertEquals("Time", metadata.getColumnName(1));
assertEquals(TSDataType.INT64, metadata.getColumnType(1));
@@ -289,17 +395,26 @@ public class TableModelEncodingCompressionCompatibilityTest {
assertEquals(TSDataType.STRING, metadata.getColumnType(2));
assertEquals(fixtureCase.valueColumn, metadata.getColumnName(3));
assertEquals(fixtureCase.dataType, metadata.getColumnType(3));
-
- int row = 0;
+ int row = start;
while (resultSet.next()) {
+ assertTrue("unexpected row in " + fixtureCase.fileName, row <= end);
assertEquals("time at row " + row, row, resultSet.getLong(1));
assertFalse("tag is null at row " + row, resultSet.isNull(2));
assertEquals(TAG_VALUE, resultSet.getString(2));
- assertFalse("value is null at row " + row, resultSet.isNull(3));
- assertValue(fixtureCase, row, resultSet);
+ assertEquals(
+ fixtureCase.fileName + " null at row " + row,
+ isNull(fixtureCase, row),
+ resultSet.isNull(3));
+ if (!isNull(fixtureCase, row)) {
+ assertValue(fixtureCase, row, resultSet);
+ }
+ if (withAnchor) {
+ assertFalse(resultSet.isNull(4));
+ assertEquals(row, resultSet.getLong(4));
+ }
row++;
}
- assertEquals(fixtureCase.rowCount, row);
+ assertEquals(fixtureCase.fileName + " row count", end + 1, row);
}
}
@@ -321,8 +436,16 @@ public class TableModelEncodingCompressionCompatibilityTest {
fixtureCase.encoding,
fixtureCase.compression,
props));
- return new TableSchema(
- fixtureCase.tableName, schemas, Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD));
+ List<ColumnCategory> categories =
+ new ArrayList<>(Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD));
+ if (fixtureCase.nullPattern != NullPattern.NONE) {
+ schemas = new ArrayList<>(schemas);
+ schemas.add(
+ new MeasurementSchema(
+ ANCHOR_COLUMN, TSDataType.INT64, TSEncoding.PLAIN, fixtureCase.compression));
+ categories.add(ColumnCategory.FIELD);
+ }
+ return new TableSchema(fixtureCase.tableName, schemas, categories);
}
private static Tablet tablet(TableSchema tableSchema, FixtureCase fixtureCase) {
@@ -336,7 +459,14 @@ public class TableModelEncodingCompressionCompatibilityTest {
for (int row = 0; row < fixtureCase.rowCount; row++) {
tablet.addTimestamp(row, row);
tablet.addValue(TAG_COLUMN, row, TAG_VALUE);
- addValue(tablet, fixtureCase, row);
+ if (isNull(fixtureCase, row)) {
+ tablet.addValue(VALUE_COLUMN, row, null);
+ } else {
+ addValue(tablet, fixtureCase, row);
+ }
+ if (fixtureCase.nullPattern != NullPattern.NONE) {
+ tablet.addValue(ANCHOR_COLUMN, row, (long) row);
+ }
}
return tablet;
}
@@ -508,6 +638,7 @@ public class TableModelEncodingCompressionCompatibilityTest {
private final TSEncoding encoding;
private final CompressionType compression;
private final int rowCount;
+ private final NullPattern nullPattern;
private FixtureCase(
TSDataType dataType, TSEncoding encoding, CompressionType compression, int rowCount) {
@@ -531,6 +662,46 @@ public class TableModelEncodingCompressionCompatibilityTest {
TSEncoding encoding,
CompressionType compression,
int rowCount) {
+ this(
+ fileName,
+ tableName,
+ tagColumn,
+ valueColumn,
+ dataType,
+ encoding,
+ compression,
+ rowCount,
+ NullPattern.NONE);
+ }
+
+ private FixtureCase(
+ TSDataType dataType, CompressionType compression, int rowCount, NullPattern nullPattern) {
+ this(
+ fileName(dataType, TSEncoding.PLAIN, compression)
+ + "."
+ + nullPattern.name().toLowerCase(Locale.ROOT)
+ + "."
+ + rowCount,
+ TABLE_NAME,
+ TAG_COLUMN,
+ VALUE_COLUMN,
+ dataType,
+ TSEncoding.PLAIN,
+ compression,
+ rowCount,
+ nullPattern);
+ }
+
+ private FixtureCase(
+ String fileName,
+ String tableName,
+ String tagColumn,
+ String valueColumn,
+ TSDataType dataType,
+ TSEncoding encoding,
+ CompressionType compression,
+ int rowCount,
+ NullPattern nullPattern) {
this.fileName = fileName;
this.tableName = tableName;
this.tagColumn = tagColumn;
@@ -539,11 +710,12 @@ public class TableModelEncodingCompressionCompatibilityTest {
this.encoding = encoding;
this.compression = compression;
this.rowCount = rowCount;
+ this.nullPattern = nullPattern;
}
private static FixtureCase fromManifestLine(String line) {
String[] parts = line.split(",", -1);
- if (parts.length != 8) {
+ if (parts.length != 9) {
throw new IllegalArgumentException("Bad manifest line: " + line);
}
return new FixtureCase(
@@ -554,7 +726,8 @@ public class TableModelEncodingCompressionCompatibilityTest {
TSDataType.valueOf(parts[4]),
TSEncoding.valueOf(parts[5]),
CompressionType.valueOf(parts[6]),
- Integer.parseInt(parts[7]));
+ Integer.parseInt(parts[7]),
+ NullPattern.valueOf(parts[8]));
}
private String toManifestLine() {
@@ -567,7 +740,8 @@ public class TableModelEncodingCompressionCompatibilityTest {
dataType.name(),
encoding.name(),
compression.name(),
- String.valueOf(rowCount));
+ String.valueOf(rowCount),
+ nullPattern.name());
}
private static String fileName(