You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ka...@apache.org on 2026/01/08 01:03:38 UTC
(doris) branch tpc_preview4-external updated (c0d3fc854cb -> db54cd036af)
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a change to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
from c0d3fc854cb add scanner profile to level 1
new 0375981938d [fix](parquet)fix parquet topn lazy mat complex data error result (#58785)
new 22931c2e922 [fix](profile) sort out parquet reader profile (#58895)
new db54cd036af [feature](vparquet-reader) Implements parquet file page cache.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.cpp | 5 +
be/src/common/config.h | 4 +
be/src/io/cache/cached_remote_file_reader.h | 2 +
be/src/io/file_factory.cpp | 15 +
be/src/io/file_factory.h | 6 +
be/src/io/fs/broker_file_reader.cpp | 6 +-
be/src/io/fs/broker_file_reader.h | 5 +-
be/src/io/fs/broker_file_system.cpp | 2 +-
be/src/io/fs/buffered_reader.cpp | 3 -
be/src/io/fs/buffered_reader.h | 28 +-
be/src/io/fs/file_reader.h | 3 +
be/src/io/fs/hdfs_file_reader.cpp | 7 +-
be/src/io/fs/hdfs_file_reader.h | 5 +-
be/src/io/fs/http_file_reader.cpp | 7 +-
be/src/io/fs/http_file_reader.h | 5 +-
be/src/io/fs/http_file_system.cpp | 2 +-
be/src/io/fs/local_file_reader.h | 2 +
be/src/io/fs/s3_file_reader.h | 2 +
be/src/io/fs/stream_load_pipe.h | 2 +
be/src/io/fs/tracing_file_reader.h | 2 +
be/src/olap/rowset/segment_v2/row_ranges.h | 10 +-
be/src/vec/exec/format/orc/orc_file_reader.h | 2 +
be/src/vec/exec/format/parquet/level_decoder.cpp | 1 +
be/src/vec/exec/format/parquet/parquet_common.cpp | 101 ++
be/src/vec/exec/format/parquet/parquet_common.h | 100 +-
.../parquet/vparquet_column_chunk_reader.cpp | 650 +++++++--
.../format/parquet/vparquet_column_chunk_reader.h | 189 ++-
.../exec/format/parquet/vparquet_column_reader.cpp | 517 ++++---
.../exec/format/parquet/vparquet_column_reader.h | 236 +++-
.../exec/format/parquet/vparquet_group_reader.cpp | 25 +-
.../exec/format/parquet/vparquet_group_reader.h | 5 +-
.../exec/format/parquet/vparquet_page_reader.cpp | 208 ++-
.../vec/exec/format/parquet/vparquet_page_reader.h | 283 ++--
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 220 +--
be/src/vec/exec/format/parquet/vparquet_reader.h | 23 +-
be/src/vec/exec/scan/file_scanner.cpp | 1 -
be/test/io/fs/buffered_reader_test.cpp | 6 +
.../format/file_reader/file_meta_cache_test.cpp | 2 +
.../format/parquet/parquet_page_cache_test.cpp | 859 ++++++++++++
.../exec/format/parquet/parquet_thrift_test.cpp | 25 +-
be/test/vec/exec/orc/orc_file_reader_test.cpp | 2 +
.../scripts/create_preinstalled_scripts/run80.hql | 21 +
.../data_part_1.parquet | Bin 0 -> 4547 bytes
.../data_part_2.parquet | Bin 0 -> 4542 bytes
.../data_part_3.parquet | Bin 0 -> 4342 bytes
.../data_part_4.parquet | Bin 0 -> 4354 bytes
.../data_part_1.parquet | Bin 0 -> 5732 bytes
.../data_part_2.parquet | Bin 0 -> 5715 bytes
.../data_part_3.parquet | Bin 0 -> 7565 bytes
.../data_part_4.parquet | Bin 0 -> 5531 bytes
.../java/org/apache/doris/qe/SessionVariable.java | 27 +
gensrc/thrift/PaloInternalService.thrift | 4 +
.../hive/test_hive_topn_lazy_mat.out | 1427 ++++++++++++++++++++
.../hive/test_hive_topn_lazy_mat.groovy | 23 +-
54 files changed, 4143 insertions(+), 937 deletions(-)
create mode 100644 be/test/vec/exec/format/parquet/parquet_page_cache_test.cpp
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet
create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/03: [fix](parquet)fix parquet topn lazy mat complex data error result (#58785)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0375981938dc435e25c2142fa458f264c432afce
Author: daidai <ch...@selectdb.com>
AuthorDate: Tue Dec 16 12:04:44 2025 +0800
[fix](parquet)fix parquet topn lazy mat complex data error result (#58785)
Related PR: #51329
Problem Summary:
This PR primarily enables the Parquet reader to use page indexes when
reading complex columns, and also fixes a data reading error in PR
of topn.
---
be/src/olap/rowset/segment_v2/row_ranges.h | 10 +-
be/src/vec/exec/format/parquet/level_decoder.cpp | 1 +
be/src/vec/exec/format/parquet/parquet_common.cpp | 101 ++
be/src/vec/exec/format/parquet/parquet_common.h | 100 +-
.../parquet/vparquet_column_chunk_reader.cpp | 333 +++--
.../format/parquet/vparquet_column_chunk_reader.h | 126 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 501 +++----
.../exec/format/parquet/vparquet_column_reader.h | 117 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 8 +-
.../exec/format/parquet/vparquet_page_reader.cpp | 65 +-
.../vec/exec/format/parquet/vparquet_page_reader.h | 209 ++-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 51 +-
.../exec/format/parquet/parquet_thrift_test.cpp | 25 +-
.../scripts/create_preinstalled_scripts/run80.hql | 21 +
.../data_part_1.parquet | Bin 0 -> 4547 bytes
.../data_part_2.parquet | Bin 0 -> 4542 bytes
.../data_part_3.parquet | Bin 0 -> 4342 bytes
.../data_part_4.parquet | Bin 0 -> 4354 bytes
.../data_part_1.parquet | Bin 0 -> 5732 bytes
.../data_part_2.parquet | Bin 0 -> 5715 bytes
.../data_part_3.parquet | Bin 0 -> 7565 bytes
.../data_part_4.parquet | Bin 0 -> 5531 bytes
.../hive/test_hive_topn_lazy_mat.out | 1427 ++++++++++++++++++++
.../hive/test_hive_topn_lazy_mat.groovy | 23 +-
24 files changed, 2406 insertions(+), 712 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/row_ranges.h b/be/src/olap/rowset/segment_v2/row_ranges.h
index b9f340823b1..2f990faa762 100644
--- a/be/src/olap/rowset/segment_v2/row_ranges.h
+++ b/be/src/olap/rowset/segment_v2/row_ranges.h
@@ -232,15 +232,15 @@ public:
return _ranges[_ranges.size() - 1].to();
}
- size_t range_size() { return _ranges.size(); }
+ size_t range_size() const { return _ranges.size(); }
- RowRange get_range(size_t index) { return _ranges[index]; }
+ RowRange get_range(size_t index) const { return _ranges[index]; }
- int64_t get_range_from(size_t range_index) { return _ranges[range_index].from(); }
+ int64_t get_range_from(size_t range_index) const { return _ranges[range_index].from(); }
- int64_t get_range_to(size_t range_index) { return _ranges[range_index].to(); }
+ int64_t get_range_to(size_t range_index) const { return _ranges[range_index].to(); }
- size_t get_range_count(size_t range_index) { return _ranges[range_index].count(); }
+ size_t get_range_count(size_t range_index) const { return _ranges[range_index].count(); }
std::string to_string() {
std::string result;
diff --git a/be/src/vec/exec/format/parquet/level_decoder.cpp b/be/src/vec/exec/format/parquet/level_decoder.cpp
index 79c18c7af08..033ca5cbdd1 100644
--- a/be/src/vec/exec/format/parquet/level_decoder.cpp
+++ b/be/src/vec/exec/format/parquet/level_decoder.cpp
@@ -87,6 +87,7 @@ doris::Status doris::vectorized::LevelDecoder::init_v2(const doris::Slice& level
}
size_t doris::vectorized::LevelDecoder::get_levels(doris::vectorized::level_t* levels, size_t n) {
+ // toto template.
if (_encoding == tparquet::Encoding::RLE) {
n = std::min((size_t)_num_levels, n);
auto num_decoded = _rle_decoder.get_values(levels, n);
diff --git a/be/src/vec/exec/format/parquet/parquet_common.cpp b/be/src/vec/exec/format/parquet/parquet_common.cpp
index 4a25102e4fe..95920ef2a16 100644
--- a/be/src/vec/exec/format/parquet/parquet_common.cpp
+++ b/be/src/vec/exec/format/parquet/parquet_common.cpp
@@ -118,6 +118,107 @@ Status FilterMap::generate_nested_filter_map(const std::vector<level_t>& rep_lev
return Status::OK();
}
+Status ColumnSelectVector::init(const std::vector<uint16_t>& run_length_null_map, size_t num_values,
+ NullMap* null_map, FilterMap* filter_map, size_t filter_map_index,
+ const std::unordered_set<size_t>* skipped_indices) {
+ _num_values = num_values;
+ _num_nulls = 0;
+ _read_index = 0;
+ size_t map_index = 0;
+ bool is_null = false;
+ _has_filter = filter_map->has_filter();
+
+ if (filter_map->has_filter()) {
+ // No run length null map is generated when _filter_all = true
+ // DCHECK(!filter_map->filter_all());
+ _data_map.resize(num_values);
+ for (auto& run_length : run_length_null_map) {
+ if (is_null) {
+ _num_nulls += run_length;
+ for (int i = 0; i < run_length; ++i) {
+ _data_map[map_index++] = FILTERED_NULL;
+ }
+ } else {
+ for (int i = 0; i < run_length; ++i) {
+ _data_map[map_index++] = FILTERED_CONTENT;
+ }
+ }
+ is_null = !is_null;
+ }
+
+ size_t num_read = 0;
+ size_t i = 0;
+ size_t valid_count = 0;
+
+ while (valid_count < num_values) {
+ DCHECK_LT(filter_map_index + i, filter_map->filter_map_size());
+
+ if (skipped_indices != nullptr && skipped_indices->count(filter_map_index + i) > 0) {
+ ++i;
+ continue;
+ }
+
+ if (filter_map->filter_map_data()[filter_map_index + i]) {
+ _data_map[valid_count] =
+ _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT;
+ num_read++;
+ }
+ ++valid_count;
+ ++i;
+ }
+
+ _num_filtered = num_values - num_read;
+
+ if (null_map != nullptr && num_read > 0) {
+ NullMap& map_data_column = *null_map;
+ auto null_map_index = map_data_column.size();
+ map_data_column.resize(null_map_index + num_read);
+
+ if (_num_nulls == 0) {
+ memset(map_data_column.data() + null_map_index, 0, num_read);
+ } else if (_num_nulls == num_values) {
+ memset(map_data_column.data() + null_map_index, 1, num_read);
+ } else {
+ for (i = 0; i < num_values; ++i) {
+ if (_data_map[i] == CONTENT) {
+ map_data_column[null_map_index++] = (UInt8) false;
+ } else if (_data_map[i] == NULL_DATA) {
+ map_data_column[null_map_index++] = (UInt8) true;
+ }
+ }
+ }
+ }
+ } else {
+ _num_filtered = 0;
+ _run_length_null_map = &run_length_null_map;
+ if (null_map != nullptr) {
+ NullMap& map_data_column = *null_map;
+ auto null_map_index = map_data_column.size();
+ map_data_column.resize(null_map_index + num_values);
+
+ for (auto& run_length : run_length_null_map) {
+ if (is_null) {
+ memset(map_data_column.data() + null_map_index, 1, run_length);
+ null_map_index += run_length;
+ _num_nulls += run_length;
+ } else {
+ memset(map_data_column.data() + null_map_index, 0, run_length);
+ null_map_index += run_length;
+ }
+ is_null = !is_null;
+ }
+ } else {
+ for (auto& run_length : run_length_null_map) {
+ if (is_null) {
+ _num_nulls += run_length;
+ }
+ is_null = !is_null;
+ }
+ }
+ }
+ return Status::OK();
+}
+
ParsedVersion::ParsedVersion(std::string application, std::optional<std::string> version,
std::optional<std::string> app_build_hash)
: _application(std::move(application)),
diff --git a/be/src/vec/exec/format/parquet/parquet_common.h b/be/src/vec/exec/format/parquet/parquet_common.h
index 293b5879108..a57ce9db7ea 100644
--- a/be/src/vec/exec/format/parquet/parquet_common.h
+++ b/be/src/vec/exec/format/parquet/parquet_common.h
@@ -92,105 +92,7 @@ public:
Status init(const std::vector<uint16_t>& run_length_null_map, size_t num_values,
NullMap* null_map, FilterMap* filter_map, size_t filter_map_index,
- const std::unordered_set<size_t>* skipped_indices = nullptr) {
- _num_values = num_values;
- _num_nulls = 0;
- _read_index = 0;
- size_t map_index = 0;
- bool is_null = false;
- _has_filter = filter_map->has_filter();
-
- if (filter_map->has_filter()) {
- // No run length null map is generated when _filter_all = true
- DCHECK(!filter_map->filter_all());
- _data_map.resize(num_values);
- for (auto& run_length : run_length_null_map) {
- if (is_null) {
- _num_nulls += run_length;
- for (int i = 0; i < run_length; ++i) {
- _data_map[map_index++] = FILTERED_NULL;
- }
- } else {
- for (int i = 0; i < run_length; ++i) {
- _data_map[map_index++] = FILTERED_CONTENT;
- }
- }
- is_null = !is_null;
- }
-
- size_t num_read = 0;
- size_t i = 0;
- size_t valid_count = 0;
-
- while (valid_count < num_values) {
- DCHECK_LT(filter_map_index + i, filter_map->filter_map_size());
-
- if (skipped_indices != nullptr &&
- skipped_indices->count(filter_map_index + i) > 0) {
- ++i;
- continue;
- }
-
- if (filter_map->filter_map_data()[filter_map_index + i]) {
- _data_map[valid_count] =
- _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT;
- num_read++;
- }
- ++valid_count;
- ++i;
- }
-
- _num_filtered = num_values - num_read;
-
- if (null_map != nullptr && num_read > 0) {
- NullMap& map_data_column = *null_map;
- auto null_map_index = map_data_column.size();
- map_data_column.resize(null_map_index + num_read);
-
- if (_num_nulls == 0) {
- memset(map_data_column.data() + null_map_index, 0, num_read);
- } else if (_num_nulls == num_values) {
- memset(map_data_column.data() + null_map_index, 1, num_read);
- } else {
- for (i = 0; i < num_values; ++i) {
- if (_data_map[i] == CONTENT) {
- map_data_column[null_map_index++] = (UInt8) false;
- } else if (_data_map[i] == NULL_DATA) {
- map_data_column[null_map_index++] = (UInt8) true;
- }
- }
- }
- }
- } else {
- _num_filtered = 0;
- _run_length_null_map = &run_length_null_map;
- if (null_map != nullptr) {
- NullMap& map_data_column = *null_map;
- auto null_map_index = map_data_column.size();
- map_data_column.resize(null_map_index + num_values);
-
- for (auto& run_length : run_length_null_map) {
- if (is_null) {
- memset(map_data_column.data() + null_map_index, 1, run_length);
- null_map_index += run_length;
- _num_nulls += run_length;
- } else {
- memset(map_data_column.data() + null_map_index, 0, run_length);
- null_map_index += run_length;
- }
- is_null = !is_null;
- }
- } else {
- for (auto& run_length : run_length_null_map) {
- if (is_null) {
- _num_nulls += run_length;
- }
- is_null = !is_null;
- }
- }
- }
- return Status::OK();
- }
+ const std::unordered_set<size_t>* skipped_indices = nullptr);
size_t num_values() const { return _num_values; }
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
index 4484e2142ee..61a25cafa26 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
@@ -47,77 +47,109 @@ struct IOContext;
namespace doris::vectorized {
#include "common/compile_check_begin.h"
-ColumnChunkReader::ColumnChunkReader(io::BufferedStreamReader* reader,
- tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema,
- const tparquet::OffsetIndex* offset_index,
- const cctz::time_zone* ctz, io::IOContext* io_ctx)
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::ColumnChunkReader(
+ io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk,
+ FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows,
+ io::IOContext* io_ctx)
: _field_schema(field_schema),
_max_rep_level(field_schema->repetition_level),
_max_def_level(field_schema->definition_level),
_stream_reader(reader),
_metadata(column_chunk->meta_data),
_offset_index(offset_index),
- // _ctz(ctz),
+ _total_rows(total_rows),
_io_ctx(io_ctx) {}
-Status ColumnChunkReader::init() {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::init() {
size_t start_offset = has_dict_page(_metadata) ? _metadata.dictionary_page_offset
: _metadata.data_page_offset;
size_t chunk_size = _metadata.total_compressed_size;
// create page reader
- _page_reader = create_page_reader(_stream_reader, _io_ctx, start_offset, chunk_size,
- _metadata.num_values, _offset_index);
+ _page_reader = create_page_reader<IN_COLLECTION, OFFSET_INDEX>(
+ _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _offset_index);
// get the block compression codec
RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec));
- if (has_dict_page(_metadata)) {
- // seek to the directory page
- _page_reader->seek_to_page(_metadata.dictionary_page_offset);
- // Parse dictionary data when reading
- // RETURN_IF_ERROR(_page_reader->next_page_header());
- // RETURN_IF_ERROR(_decode_dict_page());
- } else {
- // seek to the first data page
- _page_reader->seek_to_page(_metadata.data_page_offset);
- }
_state = INITIALIZED;
+ RETURN_IF_ERROR(_parse_first_page_header());
return Status::OK();
}
-Status ColumnChunkReader::next_page() {
- if (_state == HEADER_PARSED) {
- return Status::OK();
- }
- if (UNLIKELY(_state == NOT_INIT)) {
- return Status::Corruption("Should initialize chunk reader");
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::skip_nested_values(
+ const std::vector<level_t>& def_levels) {
+ size_t no_value_cnt = 0;
+ size_t value_cnt = 0;
+
+ for (size_t idx = 0; idx < def_levels.size(); idx++) {
+ level_t def_level = def_levels[idx];
+ if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) {
+ no_value_cnt++;
+ } else if (def_level < _field_schema->definition_level) {
+ no_value_cnt++;
+ } else {
+ value_cnt++;
+ }
}
- if (UNLIKELY(_remaining_num_values != 0)) {
- return Status::Corruption("Should skip current page");
+
+ RETURN_IF_ERROR(skip_values(value_cnt, true));
+ RETURN_IF_ERROR(skip_values(no_value_cnt, false));
+ return Status::OK();
+}
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_parse_first_page_header() {
+ RETURN_IF_ERROR(parse_page_header());
+
+ const tparquet::PageHeader* header = nullptr;
+ RETURN_IF_ERROR(_page_reader->get_page_header(header));
+ if (header->type == tparquet::PageType::DICTIONARY_PAGE) {
+ // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false,
+ // so we should parse the directory page in next_page()
+ RETURN_IF_ERROR(_decode_dict_page());
+ // parse the real first data page
+ RETURN_IF_ERROR(_page_reader->dict_next_page());
+ _state = INITIALIZED;
}
- RETURN_IF_ERROR(_page_reader->next_page_header());
-
- if (!_dict_checked) {
- _dict_checked = true;
- const tparquet::PageHeader* header;
- RETURN_IF_ERROR(_page_reader->get_page_header(header));
- if (header->type == tparquet::PageType::DICTIONARY_PAGE) {
- // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false,
- // so we should parse the directory page in next_page()
- RETURN_IF_ERROR(_decode_dict_page());
- // parse the real first data page
- return next_page();
- }
+ return Status::OK();
+}
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
+ if (_state == HEADER_PARSED || _state == DATA_LOADED) {
+ return Status::OK();
}
+ RETURN_IF_ERROR(_page_reader->parse_page_header());
- RETURN_IF_ERROR(_page_reader->get_num_values(_remaining_num_values));
- _chunk_parsed_values += _remaining_num_values;
+ const tparquet::PageHeader* header = nullptr;
+ ;
+ RETURN_IF_ERROR(_page_reader->get_page_header(header));
+ int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values
+ : header->data_page_header.num_values;
+ _remaining_rep_nums = page_num_values;
+ _remaining_def_nums = page_num_values;
+ _remaining_num_values = page_num_values;
+
+ // no offset will parse all header.
+ if constexpr (OFFSET_INDEX == false) {
+ _chunk_parsed_values += _remaining_num_values;
+ }
_state = HEADER_PARSED;
+ return Status::OK();
+}
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::next_page() {
+ _state = INITIALIZED;
+ RETURN_IF_ERROR(_page_reader->next_page());
return Status::OK();
}
-void ColumnChunkReader::_get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2,
- Slice& page_data) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+void ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_get_uncompressed_levels(
+ const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) {
int32_t rl = page_v2.repetition_levels_byte_length;
int32_t dl = page_v2.definition_levels_byte_length;
_v2_rep_levels = Slice(page_data.data, rl);
@@ -126,12 +158,16 @@ void ColumnChunkReader::_get_uncompressed_levels(const tparquet::DataPageHeaderV
page_data.size -= dl + rl;
}
-Status ColumnChunkReader::load_page_data() {
- // TODO: remove checking HEADER_PARSED or change name
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
+ if (_state == DATA_LOADED) {
+ return Status::OK();
+ }
if (UNLIKELY(_state != HEADER_PARSED)) {
return Status::Corruption("Should parse page header");
}
- const tparquet::PageHeader* header;
+
+ const tparquet::PageHeader* header = nullptr;
RETURN_IF_ERROR(_page_reader->get_page_header(header));
int32_t uncompressed_size = header->uncompressed_page_size;
@@ -171,22 +207,22 @@ Status ColumnChunkReader::load_page_data() {
SCOPED_RAW_TIMER(&_statistics.decode_level_time);
if (header->__isset.data_page_header_v2) {
RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level,
- _remaining_num_values));
+ _remaining_rep_nums));
} else {
RETURN_IF_ERROR(_rep_level_decoder.init(
&_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level,
- _remaining_num_values));
+ _remaining_rep_nums));
}
}
if (_max_def_level > 0) {
SCOPED_RAW_TIMER(&_statistics.decode_level_time);
if (header->__isset.data_page_header_v2) {
RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level,
- _remaining_num_values));
+ _remaining_def_nums));
} else {
RETURN_IF_ERROR(_def_level_decoder.init(
&_page_data, header->data_page_header.definition_level_encoding, _max_def_level,
- _remaining_num_values));
+ _remaining_def_nums));
}
}
auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding
@@ -204,8 +240,6 @@ Status ColumnChunkReader::load_page_data() {
RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder));
// Set type length
page_decoder->set_type_length(_get_type_length());
- // Initialize the time convert context
- // page_decoder->init(_field_schema, _ctz);
_decoders[static_cast<int>(encoding)] = std::move(page_decoder);
_page_decoder = _decoders[static_cast<int>(encoding)].get();
}
@@ -216,8 +250,9 @@ Status ColumnChunkReader::load_page_data() {
return Status::OK();
}
-Status ColumnChunkReader::_decode_dict_page() {
- const tparquet::PageHeader* header;
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_decode_dict_page() {
+ const tparquet::PageHeader* header = nullptr;
RETURN_IF_ERROR(_page_reader->get_page_header(header));
DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type);
SCOPED_RAW_TIMER(&_statistics.decode_dict_time);
@@ -253,8 +288,6 @@ Status ColumnChunkReader::_decode_dict_page() {
Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder));
// Set type length
page_decoder->set_type_length(_get_type_length());
- // Initialize the time convert context
- // page_decoder->init(_field_schema, _ctz);
// Set the dictionary data
RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size,
header->dictionary_page_header.num_values));
@@ -264,14 +297,17 @@ Status ColumnChunkReader::_decode_dict_page() {
return Status::OK();
}
-void ColumnChunkReader::_reserve_decompress_buf(size_t size) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+void ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_reserve_decompress_buf(size_t size) {
if (size > _decompress_buf_size) {
_decompress_buf_size = BitUtil::next_power_of_two(size);
_decompress_buf = make_unique_buffer<uint8_t>(_decompress_buf_size);
}
}
-Status ColumnChunkReader::skip_values(size_t num_values, bool skip_data) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::skip_values(size_t num_values,
+ bool skip_data) {
if (UNLIKELY(_remaining_num_values < num_values)) {
return Status::IOError("Skip too many values in current page. {} vs. {}",
_remaining_num_values, num_values);
@@ -285,24 +321,10 @@ Status ColumnChunkReader::skip_values(size_t num_values, bool skip_data) {
}
}
-void ColumnChunkReader::insert_null_values(MutableColumnPtr& doris_column, size_t num_values) {
- SCOPED_RAW_TIMER(&_statistics.decode_value_time);
- doris_column->insert_many_defaults(num_values);
- _remaining_num_values -= num_values;
-}
-
-size_t ColumnChunkReader::get_rep_levels(level_t* levels, size_t n) {
- DCHECK_GT(_max_rep_level, 0);
- return _rep_level_decoder.get_levels(levels, n);
-}
-
-size_t ColumnChunkReader::get_def_levels(level_t* levels, size_t n) {
- DCHECK_GT(_max_def_level, 0);
- return _def_level_decoder.get_levels(levels, n);
-}
-
-Status ColumnChunkReader::decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type,
- ColumnSelectVector& select_vector, bool is_dict_filter) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::decode_values(
+ MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector,
+ bool is_dict_filter) {
if (select_vector.num_values() == 0) {
return Status::OK();
}
@@ -317,7 +339,159 @@ Status ColumnChunkReader::decode_values(MutableColumnPtr& doris_column, DataType
return _page_decoder->decode_values(doris_column, data_type, select_vector, is_dict_filter);
}
-int32_t ColumnChunkReader::_get_type_length() {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::seek_to_nested_row(size_t left_row) {
+ if constexpr (OFFSET_INDEX) {
+ while (true) {
+ if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) {
+ break;
+ } else if (has_next_page()) {
+ RETURN_IF_ERROR(next_page());
+ _current_row = _page_reader->start_row();
+ } else [[unlikely]] {
+ return Status::InternalError("no match seek row {}, current row {}", left_row,
+ _current_row);
+ }
+ };
+
+ RETURN_IF_ERROR(parse_page_header());
+ RETURN_IF_ERROR(load_page_data());
+ RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row));
+ _current_row = left_row;
+ } else {
+ while (true) {
+ RETURN_IF_ERROR(parse_page_header());
+ if (_page_reader->is_header_v2()) {
+ if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) {
+ RETURN_IF_ERROR(load_page_data());
+ // this page contain this row.
+ RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row));
+ _current_row = left_row;
+ break;
+ }
+
+ _current_row = _page_reader->end_row();
+ if (has_next_page()) [[likely]] {
+ RETURN_IF_ERROR(next_page());
+ } else {
+ return Status::InternalError("no match seek row {}, current row {}", left_row,
+ _current_row);
+ }
+ } else {
+ RETURN_IF_ERROR(load_page_data());
+ std::vector<level_t> rep_levels;
+ std::vector<level_t> def_levels;
+ bool cross_page = false;
+
+ size_t result_rows = 0;
+ RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row,
+ &result_rows, &cross_page));
+ RETURN_IF_ERROR(fill_def(def_levels));
+ RETURN_IF_ERROR(skip_nested_values(def_levels));
+ bool need_load_next_page = true;
+ while (cross_page) {
+ need_load_next_page = false;
+ rep_levels.clear();
+ def_levels.clear();
+ RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page));
+ RETURN_IF_ERROR(fill_def(def_levels));
+ RETURN_IF_ERROR(skip_nested_values(def_levels));
+ }
+ if (left_row == _current_row) {
+ break;
+ }
+ if (need_load_next_page) {
+ if (has_next_page()) [[likely]] {
+ RETURN_IF_ERROR(next_page());
+ } else {
+ return Status::InternalError("no match seek row {}, current row {}",
+ left_row, _current_row);
+ }
+ }
+ }
+ };
+ }
+
+ return Status::OK();
+}
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_skip_nested_rows_in_page(size_t num_rows) {
+ if (num_rows == 0) {
+ return Status::OK();
+ }
+
+ std::vector<level_t> rep_levels;
+ std::vector<level_t> def_levels;
+
+ bool cross_page = false;
+ size_t result_rows = 0;
+ RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page));
+ RETURN_IF_ERROR(fill_def(def_levels));
+ RETURN_IF_ERROR(skip_nested_values(def_levels));
+ DCHECK(cross_page == false);
+ if (num_rows != result_rows) [[unlikely]] {
+ return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows,
+ result_rows);
+ }
+ return Status::OK();
+}
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_nested_rows(
+ std::vector<level_t>& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) {
+ if (_state != DATA_LOADED) [[unlikely]] {
+ return Status::IOError("Should load page data first to load nested rows");
+ }
+ *cross_page = false;
+ *result_rows = 0;
+ rep_levels.reserve(rep_levels.size() + _remaining_rep_nums);
+ while (_remaining_rep_nums) {
+ level_t rep_level = _rep_level_decoder.get_next();
+ if (rep_level == 0) { // rep_level 0 indicates start of new row
+ if (*result_rows == max_rows) { // this page contain max_rows, page no end.
+ _current_row += max_rows;
+ _rep_level_decoder.rewind_one();
+ return Status::OK();
+ }
+ (*result_rows)++;
+ }
+ _remaining_rep_nums--;
+ rep_levels.emplace_back(rep_level);
+ }
+ _current_row += *result_rows;
+
+ auto need_check_cross_page = [&]() -> bool {
+ return !OFFSET_INDEX && _remaining_rep_nums == 0 && !_page_reader->is_header_v2() &&
+ has_next_page();
+ };
+ *cross_page = need_check_cross_page();
+ return Status::OK();
+};
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_cross_page_nested_row(
+ std::vector<level_t>& rep_levels, bool* cross_page) {
+ RETURN_IF_ERROR(next_page());
+ RETURN_IF_ERROR(parse_page_header());
+ RETURN_IF_ERROR(load_page_data());
+
+ *cross_page = has_next_page();
+ while (_remaining_rep_nums) {
+ level_t rep_level = _rep_level_decoder.get_next();
+ if (rep_level == 0) { // rep_level 0 indicates start of new row
+ *cross_page = false;
+ _rep_level_decoder.rewind_one();
+ break;
+ }
+ _remaining_rep_nums--;
+ rep_levels.emplace_back(rep_level);
+ }
+ return Status::OK();
+}
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+int32_t ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_get_type_length() {
switch (_field_schema->physical_type) {
case tparquet::Type::INT32:
[[fallthrough]];
@@ -355,5 +529,10 @@ bool has_dict_page(const tparquet::ColumnMetaData& column) {
column.dictionary_page_offset < column.data_page_offset;
}
+template class ColumnChunkReader<true, true>;
+template class ColumnChunkReader<true, false>;
+template class ColumnChunkReader<false, true>;
+template class ColumnChunkReader<false, false>;
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
index f8e182d6583..992bc823430 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
@@ -34,9 +34,6 @@
#include "vec/exec/format/parquet/parquet_common.h"
#include "vparquet_page_reader.h"
-namespace cctz {
-class time_zone;
-} // namespace cctz
namespace doris {
class BlockCompressionCodec;
@@ -54,9 +51,29 @@ template <typename T>
class ColumnStr;
using ColumnString = ColumnStr<UInt32>;
+struct ColumnChunkReaderStatistics {
+ int64_t decompress_time = 0;
+ int64_t decompress_cnt = 0;
+ int64_t decode_header_time = 0;
+ int64_t decode_value_time = 0;
+ int64_t decode_dict_time = 0;
+ int64_t decode_level_time = 0;
+ int64_t skip_page_header_num = 0;
+ int64_t parse_page_header_num = 0;
+};
+
/**
* Read and decode parquet column data into doris block column.
- * <p>Usage:</p>
+ * <p>Usage:</p> struct ColumnChunkReaderStatistics {
+ int64_t decompress_time = 0;
+ int64_t decompress_cnt = 0;
+ int64_t decode_header_time = 0;
+ int64_t decode_value_time = 0;
+ int64_t decode_dict_time = 0;
+ int64_t decode_level_time = 0;
+ int64_t skip_page_header_num = 0;
+ int64_t parse_page_header_num = 0;
+ };
* // Create chunk reader
* ColumnChunkReader chunk_reader(BufferedStreamReader* reader,
* tparquet::ColumnChunk* column_chunk,
@@ -74,6 +91,7 @@ using ColumnString = ColumnStr<UInt32>;
* chunk_reader.decode_values(slice, num_values);
* }
*/
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
class ColumnChunkReader {
public:
struct Statistics {
@@ -90,43 +108,22 @@ public:
ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk,
FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index,
- const cctz::time_zone* ctz, io::IOContext* io_ctx);
+ size_t total_row, io::IOContext* io_ctx);
~ColumnChunkReader() = default;
// Initialize chunk reader, will generate the decoder and codec.
Status init();
// Whether the chunk reader has a more page to read.
- bool has_next_page() const { return _chunk_parsed_values < _metadata.num_values; }
-
- // Deprecated
- // Seek to the specific page, page_header_offset must be the start offset of the page header.
- // _end_offset may exceed the actual data area, so we can only use the number of parsed values
- // to determine whether there are remaining pages to read. That's to say we can't use the
- // PageLocation in parquet metadata to seek to the specified page. We should call next_page()
- // and skip_page() to skip pages one by one.
- // todo: change this interface to seek_to_page(int64_t page_header_offset, size_t num_parsed_values)
- // and set _chunk_parsed_values = num_parsed_values
- // [[deprecated]]
- void seek_to_page(int64_t page_header_offset) {
- _remaining_num_values = 0;
- _page_reader->seek_to_page(page_header_offset);
- _state = INITIALIZED;
- }
-
- // Seek to next page. Only read and parse the page header.
- Status next_page();
-
- // Skip current page(will not read and parse) if the page is filtered by predicates.
- Status skip_page() {
- Status res = Status::OK();
- _remaining_num_values = 0;
- if (_state == HEADER_PARSED) {
- res = _page_reader->skip_page();
+ bool has_next_page() const {
+ if constexpr (OFFSET_INDEX) {
+ return _page_reader->has_next_page();
+ } else {
+ // no offset need parse all page header.
+ return _chunk_parsed_values < _metadata.num_values;
}
- _state = PAGE_SKIPPED;
- return res;
}
+
// Skip some values(will not read and parse) in current page if the values are filtered by predicates.
// when skip_data = false, the underlying decoder will not skip data,
// only used when maintaining the consistency of _remaining_num_values.
@@ -143,16 +140,6 @@ public:
}
// The remaining number of values in current page(including null values). Decreased when reading or skipping.
uint32_t remaining_num_values() const { return _remaining_num_values; }
- // null values are generated from definition levels
- // the caller should maintain the consistency after analyzing null values from definition levels.
- void insert_null_values(MutableColumnPtr& doris_column, size_t num_values);
- // Get the raw data of current page.
- Slice& get_page_data() { return _page_data; }
-
- // Get the repetition levels
- size_t get_rep_levels(level_t* levels, size_t n);
- // Get the definition levels
- size_t get_def_levels(level_t* levels, size_t n);
// Decode values in current page into doris column.
Status decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type,
@@ -171,7 +158,7 @@ public:
// Get page decoder
Decoder* get_page_decoder() { return _page_decoder; }
- Statistics& statistics() {
+ ColumnChunkReaderStatistics& statistics() {
_statistics.decode_header_time = _page_reader->statistics().decode_header_time;
_statistics.skip_page_header_num = _page_reader->statistics().skip_page_header_num;
_statistics.parse_page_header_num = _page_reader->statistics().parse_page_header_num;
@@ -189,33 +176,70 @@ public:
->convert_dict_column_to_string_column(dict_column);
}
+ size_t page_start_row() const { return _page_reader->start_row(); }
+
+ size_t page_end_row() const { return _page_reader->end_row(); }
+
+ Status parse_page_header();
+ Status next_page();
+
+ Status seek_to_nested_row(size_t left_row);
+ Status skip_nested_values(const std::vector<level_t>& def_levels);
+ Status fill_def(std::vector<level_t>& def_values) {
+ auto before_sz = def_values.size();
+ auto append_sz = _remaining_def_nums - _remaining_rep_nums;
+ def_values.resize(before_sz + append_sz, 0);
+ if (max_def_level() != 0) {
+ auto ptr = def_values.data() + before_sz;
+ _def_level_decoder.get_levels(ptr, append_sz);
+ }
+ _remaining_def_nums -= append_sz;
+ return Status::OK();
+ }
+
+ Status load_page_nested_rows(std::vector<level_t>& rep_levels, size_t max_rows,
+ size_t* result_rows, bool* cross_page);
+ Status load_cross_page_nested_row(std::vector<level_t>& rep_levels, bool* cross_page);
+
private:
enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED };
+ // for check dict page.
+ Status _parse_first_page_header();
Status _decode_dict_page();
+
void _reserve_decompress_buf(size_t size);
int32_t _get_type_length();
+
void _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data);
+ Status _skip_nested_rows_in_page(size_t num_rows);
ColumnChunkReaderState _state = NOT_INIT;
FieldSchema* _field_schema = nullptr;
- level_t _max_rep_level;
- level_t _max_def_level;
- tparquet::LogicalType _parquet_logical_type;
+ const level_t _max_rep_level;
+ const level_t _max_def_level;
io::BufferedStreamReader* _stream_reader = nullptr;
tparquet::ColumnMetaData _metadata;
- const tparquet::OffsetIndex* _offset_index;
- // cctz::time_zone* _ctz;
+ const tparquet::OffsetIndex* _offset_index = nullptr;
+ size_t _current_row = 0;
+ size_t _total_rows = 0;
io::IOContext* _io_ctx = nullptr;
- std::unique_ptr<PageReader> _page_reader;
+ std::unique_ptr<PageReader<IN_COLLECTION, OFFSET_INDEX>> _page_reader;
BlockCompressionCodec* _block_compress_codec = nullptr;
LevelDecoder _rep_level_decoder;
LevelDecoder _def_level_decoder;
size_t _chunk_parsed_values = 0;
+ // this page remaining rep/def nums
+ // if max_rep_level = 0 / max_def_level = 0, this value retail hava value.
+ uint32_t _remaining_rep_nums = 0;
+ uint32_t _remaining_def_nums = 0;
+ // this page remaining values to be processed (for read/skip).
+ // need parse this page header.
uint32_t _remaining_num_values = 0;
+
Slice _page_data;
DorisUniqueBufferPtr<uint8_t> _decompress_buf;
size_t _decompress_buf_size = 0;
@@ -227,7 +251,7 @@ private:
// Map: encoding -> Decoder
// Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding
std::unordered_map<int, std::unique_ptr<Decoder>> _decoders;
- Statistics _statistics;
+ ColumnChunkReaderStatistics _statistics;
};
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index 71bfdce85c9..db6042ce703 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -108,16 +108,18 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
const tparquet::RowGroup& row_group, const RowRanges& row_ranges,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index,
- const std::set<uint64_t>& column_ids,
+ size_t max_buf_size,
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ bool in_collection, const std::set<uint64_t>& column_ids,
const std::set<uint64_t>& filter_column_ids) {
+ size_t total_rows = row_group.num_rows;
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
- element_reader, max_buf_size, nullptr, column_ids,
+ element_reader, max_buf_size, col_offsets, true, column_ids,
filter_column_ids));
- element_reader->set_nested_column();
- auto array_reader = ArrayColumnReader::create_unique(row_ranges, ctz, io_ctx);
+ // element_reader->set_nested_column();
+ auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx);
RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field));
array_reader->_filter_column_ids = filter_column_ids;
reader.reset(array_reader.release());
@@ -129,12 +131,12 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
column_ids.find(field->children[0].get_column_id()) != column_ids.end()) {
// Create key reader
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
- key_reader, max_buf_size, nullptr, column_ids,
+ key_reader, max_buf_size, col_offsets, true, column_ids,
filter_column_ids));
- key_reader->set_nested_column();
+ // key_reader->set_nested_column();
} else {
- auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx,
- &field->children[0]);
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, total_rows, ctz,
+ io_ctx, &field->children[0]);
key_reader = std::move(skip_reader);
}
@@ -142,16 +144,16 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
column_ids.find(field->children[1].get_column_id()) != column_ids.end()) {
// Create value reader
RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx,
- value_reader, max_buf_size, nullptr, column_ids,
+ value_reader, max_buf_size, col_offsets, true, column_ids,
filter_column_ids));
- value_reader->set_nested_column();
+ // value_reader->set_nested_column();
} else {
- auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx,
- &field->children[0]);
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, total_rows, ctz,
+ io_ctx, &field->children[0]);
value_reader = std::move(skip_reader);
}
- auto map_reader = MapColumnReader::create_unique(row_ranges, ctz, io_ctx);
+ auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx);
RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field));
map_reader->_filter_column_ids = filter_column_ids;
reader.reset(map_reader.release());
@@ -164,17 +166,17 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
std::unique_ptr<ParquetColumnReader> child_reader;
if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) {
RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx,
- child_reader, max_buf_size, nullptr, column_ids,
- filter_column_ids));
- child_reader->set_nested_column();
+ child_reader, max_buf_size, col_offsets, in_collection,
+ column_ids, filter_column_ids));
+ // child_reader->set_nested_column();
child_readers[child.name] = std::move(child_reader);
// Record the first non-SkippingReader
if (non_skip_reader_idx == -1) {
non_skip_reader_idx = i;
}
} else {
- auto skip_reader =
- std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, &child);
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, total_rows, ctz,
+ io_ctx, &child);
skip_reader->_filter_column_ids = filter_column_ids;
child_readers[child.name] = std::move(skip_reader);
}
@@ -183,22 +185,57 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
if (non_skip_reader_idx == -1) {
std::unique_ptr<ParquetColumnReader> child_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
- child_reader, max_buf_size, nullptr, column_ids,
- filter_column_ids));
- child_reader->set_nested_column();
+ child_reader, max_buf_size, col_offsets, in_collection,
+ column_ids, filter_column_ids));
+ // child_reader->set_nested_column();
child_readers[field->children[0].name] = std::move(child_reader);
}
- auto struct_reader = StructColumnReader::create_unique(row_ranges, ctz, io_ctx);
+ auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx);
RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field));
struct_reader->_filter_column_ids = filter_column_ids;
reader.reset(struct_reader.release());
} else {
- const tparquet::ColumnChunk& chunk = row_group.columns[field->physical_column_index];
- auto scalar_reader =
- ScalarColumnReader::create_unique(row_ranges, chunk, offset_index, ctz, io_ctx);
- RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
- scalar_reader->_filter_column_ids = filter_column_ids;
- reader.reset(scalar_reader.release());
+ auto physical_index = field->physical_column_index;
+ const tparquet::OffsetIndex* offset_index =
+ col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index]
+ : nullptr;
+
+ const tparquet::ColumnChunk& chunk = row_group.columns[physical_index];
+
+ // ScalarColumnReader::create_unique(row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
+ if (in_collection) {
+ if (offset_index == nullptr) {
+ auto scalar_reader = ScalarColumnReader<true, false>::create_unique(
+ row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
+
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
+ reader.reset(scalar_reader.release());
+ } else {
+ auto scalar_reader = ScalarColumnReader<true, true>::create_unique(
+ row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
+
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
+ reader.reset(scalar_reader.release());
+ }
+ } else {
+ if (offset_index == nullptr) {
+ auto scalar_reader = ScalarColumnReader<false, false>::create_unique(
+ row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
+
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
+ reader.reset(scalar_reader.release());
+ } else {
+ auto scalar_reader = ScalarColumnReader<false, true>::create_unique(
+ row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
+
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
+ reader.reset(scalar_reader.release());
+ }
+ }
}
return Status::OK();
}
@@ -206,13 +243,13 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
void ParquetColumnReader::_generate_read_ranges(RowRange page_row_range,
RowRanges* result_ranges) const {
result_ranges->add(page_row_range);
- if (_nested_column) {
- return;
- }
RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges);
}
-Status ScalarColumnReader::init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::init(io::FileReaderSPtr file,
+ FieldSchema* field,
+ size_t max_buf_size) {
_field_schema = field;
auto& chunk_meta = _chunk_meta.meta_data;
int64_t chunk_start = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset
@@ -228,13 +265,14 @@ Status ScalarColumnReader::init(io::FileReaderSPtr file, FieldSchema* field, siz
}
_stream_reader = std::make_unique<io::BufferedFileStreamReader>(file, chunk_start, chunk_len,
prefetch_buffer_size);
- _chunk_reader = std::make_unique<ColumnChunkReader>(_stream_reader.get(), &_chunk_meta, field,
- _offset_index, _ctz, _io_ctx);
+ _chunk_reader = std::make_unique<ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>>(
+ _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx);
RETURN_IF_ERROR(_chunk_reader->init());
return Status::OK();
}
-Status ScalarColumnReader::_skip_values(size_t num_values) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::_skip_values(size_t num_values) {
if (num_values == 0) {
return Status::OK();
}
@@ -257,7 +295,7 @@ Status ScalarColumnReader::_skip_values(size_t num_values) {
LOG(WARNING) << ss.str();
return Status::InternalError("Failed to decode definition level.");
}
- if (def_level == 0) {
+ if (def_level < _field_schema->definition_level) {
null_size += loop_skip;
} else {
nonnull_size += loop_skip;
@@ -276,9 +314,12 @@ Status ScalarColumnReader::_skip_values(size_t num_values) {
return Status::OK();
}
-Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_column,
- DataTypePtr& type, FilterMap& filter_map,
- bool is_dict_filter) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::_read_values(size_t num_values,
+ ColumnPtr& doris_column,
+ DataTypePtr& type,
+ FilterMap& filter_map,
+ bool is_dict_filter) {
if (num_values == 0) {
return Status::OK();
}
@@ -312,7 +353,11 @@ Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_colu
LOG(WARNING) << ss.str();
return Status::InternalError("Failed to decode definition level.");
}
- bool is_null = def_level == 0;
+
+ for (int i = 0; i < loop_read; i++) {
+ _def_levels.emplace_back(def_level);
+ }
+ bool is_null = def_level < _field_schema->definition_level;
if (!(prev_is_null ^ is_null)) {
null_map.emplace_back(0);
}
@@ -326,11 +371,14 @@ Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_colu
prev_is_null = is_null;
has_read += loop_read;
}
+ } else {
+ _def_levels.resize(_def_levels.size() + num_values, 0);
}
} else {
if (_chunk_reader->max_def_level() > 0) {
return Status::Corruption("Not nullable column has null values in parquet file");
}
+ _def_levels.resize(_def_levels.size() + num_values, 0);
data_column = doris_column->assume_mutable();
}
if (null_map.size() == 0) {
@@ -357,93 +405,15 @@ Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_colu
* A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that
* whether the reader should read the remaining value of the last row in previous page.
*/
-Status ScalarColumnReader::_read_nested_column(ColumnPtr& doris_column, DataTypePtr& type,
- FilterMap& filter_map, size_t batch_size,
- size_t* read_rows, bool* eof, bool is_dict_filter,
- bool align_rows) {
- std::unique_ptr<FilterMap> nested_filter_map;
-
- FilterMap* current_filter_map = &filter_map;
- size_t origin_size = 0;
- if (align_rows) {
- origin_size = _rep_levels.size();
- // just read the remaining values of the last row in previous page,
- // so there's no a new row should be read.
- batch_size = 0;
- /*
- * Since the function is repeatedly called to fetch data for the batch size,
- * it causes `_rep_levels.resize(0); _def_levels.resize(0);`, resulting in the
- * definition and repetition levels of the reader only containing the latter
- * part of the batch (i.e., missing some parts). Therefore, when using the
- * definition and repetition levels to fill the null_map for structs and maps,
- * the function should not be called multiple times before filling.
- * todo:
- * We may need to consider reading the entire batch of data at once, as this approach
- * would be more user-friendly in terms of function usage. However, we must consider that if the
- * data spans multiple pages, memory usage may increase significantly.
- */
- } else {
- _rep_levels.resize(0);
- _def_levels.resize(0);
- if (_nested_filter_map_data) {
- _nested_filter_map_data->resize(0);
- }
- }
- size_t parsed_rows = 0;
- size_t remaining_values = _chunk_reader->remaining_num_values();
- bool has_rep_level = _chunk_reader->max_rep_level() > 0;
- bool has_def_level = _chunk_reader->max_def_level() > 0;
-
- // Handle repetition levels (indicates nesting structure)
- if (has_rep_level) {
- LevelDecoder& rep_decoder = _chunk_reader->rep_level_decoder();
- // Read repetition levels until batch is full or no more values
- while (parsed_rows <= batch_size && remaining_values > 0) {
- level_t rep_level = rep_decoder.get_next();
- if (rep_level == 0) { // rep_level 0 indicates start of new row
- if (parsed_rows == batch_size) {
- rep_decoder.rewind_one();
- break;
- }
- parsed_rows++;
- }
- _rep_levels.emplace_back(rep_level);
- remaining_values--;
- }
-
- // Generate nested filter map
- if (filter_map.has_filter() && (!filter_map.filter_all())) {
- if (_nested_filter_map_data == nullptr) {
- _nested_filter_map_data.reset(new std::vector<uint8_t>());
- }
- RETURN_IF_ERROR(filter_map.generate_nested_filter_map(
- _rep_levels, *_nested_filter_map_data, &nested_filter_map,
- &_orig_filter_map_index, origin_size));
- // Update current_filter_map to nested_filter_map
- current_filter_map = nested_filter_map.get();
- }
- } else if (!align_rows) {
- // case : required child columns in struct type
- parsed_rows = std::min(remaining_values, batch_size);
- remaining_values -= parsed_rows;
- _rep_levels.resize(parsed_rows, 0);
- }
-
- // Process definition levels (indicates null values)
- size_t parsed_values = _chunk_reader->remaining_num_values() - remaining_values;
- _def_levels.resize(origin_size + parsed_values);
- if (has_def_level) {
- // if parsed_values is 0, we don't need to decode levels
- if (parsed_values != 0) {
- _chunk_reader->def_level_decoder().get_levels(&_def_levels[origin_size], parsed_values);
- }
- } else {
- std::fill(_def_levels.begin() + origin_size, _def_levels.end(), 0);
- }
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::_read_nested_column(
+ ColumnPtr& doris_column, DataTypePtr& type, FilterMap& filter_map, size_t batch_size,
+ size_t* read_rows, bool* eof, bool is_dict_filter) {
+ _rep_levels.clear();
+ _def_levels.clear();
// Handle nullable columns
MutableColumnPtr data_column;
- std::vector<uint16_t> null_map;
NullMap* map_data_column = nullptr;
if (doris_column->is_nullable()) {
SCOPED_RAW_TIMER(&_decode_null_map_time);
@@ -460,138 +430,86 @@ Status ScalarColumnReader::_read_nested_column(ColumnPtr& doris_column, DataType
data_column = doris_column->assume_mutable();
}
- // Process definition levels to build null map
- size_t has_read = origin_size;
- size_t ancestor_nulls = 0;
- size_t null_size = 0;
- size_t nonnull_size = 0;
- null_map.emplace_back(0);
- bool prev_is_null = false;
+ std::vector<uint16_t> null_map;
std::unordered_set<size_t> ancestor_null_indices;
+ std::vector<uint8_t> nested_filter_map_data;
- while (has_read < origin_size + parsed_values) {
- level_t def_level = _def_levels[has_read++];
- size_t loop_read = 1;
- while (has_read < origin_size + parsed_values && _def_levels[has_read] == def_level) {
- has_read++;
- loop_read++;
+ auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) {
+ RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels));
+ std::unique_ptr<FilterMap> nested_filter_map = std::make_unique<FilterMap>();
+ if (filter_map.has_filter()) {
+ RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz,
+ _rep_levels.size(), nested_filter_map_data,
+ &nested_filter_map));
}
- if (def_level < _field_schema->repeated_parent_def_level) {
- for (size_t i = 0; i < loop_read; i++) {
- ancestor_null_indices.insert(has_read - loop_read + i);
- }
- ancestor_nulls += loop_read;
- continue;
- }
-
- bool is_null = def_level < _field_schema->definition_level;
- if (is_null) {
- null_size += loop_read;
- } else {
- nonnull_size += loop_read;
- }
+ null_map.clear();
+ ancestor_null_indices.clear();
+ RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), null_map,
+ ancestor_null_indices));
- if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) {
- null_map.back() += loop_read;
- } else {
- if (!(prev_is_null ^ is_null)) {
- null_map.emplace_back(0);
- }
- size_t remaining = loop_read;
- while (remaining > USHRT_MAX) {
- null_map.emplace_back(USHRT_MAX);
- null_map.emplace_back(0);
- remaining -= USHRT_MAX;
- }
- null_map.emplace_back((u_short)remaining);
- prev_is_null = is_null;
- }
- }
-
- size_t num_values = parsed_values - ancestor_nulls;
-
- // Handle filtered values
- if (current_filter_map->filter_all()) {
- // Skip all values if everything is filtered
- if (null_size > 0) {
- RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false));
- }
- if (nonnull_size > 0) {
- RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true));
- }
- if (ancestor_nulls != 0) {
- RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_nulls, false));
- }
- } else {
ColumnSelectVector select_vector;
{
SCOPED_RAW_TIMER(&_decode_null_map_time);
- RETURN_IF_ERROR(
- select_vector.init(null_map, num_values, map_data_column, current_filter_map,
- _nested_filter_map_data ? origin_size : _filter_map_index,
- &ancestor_null_indices));
+ RETURN_IF_ERROR(select_vector.init(
+ null_map,
+ _rep_levels.size() - before_rep_level_sz - ancestor_null_indices.size(),
+ map_data_column, nested_filter_map.get(), 0, &ancestor_null_indices));
}
RETURN_IF_ERROR(
_chunk_reader->decode_values(data_column, type, select_vector, is_dict_filter));
- if (ancestor_nulls != 0) {
- RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_nulls, false));
- }
- }
- *read_rows += parsed_rows;
- _filter_map_index += parsed_values;
-
- // Handle cross-page reading
- if (_chunk_reader->remaining_num_values() == 0) {
- if (_chunk_reader->has_next_page()) {
- RETURN_IF_ERROR(_chunk_reader->next_page());
- RETURN_IF_ERROR(_chunk_reader->load_page_data());
- return _read_nested_column(doris_column, type, filter_map, 0, read_rows, eof,
- is_dict_filter, true);
- } else {
- *eof = true;
- }
- }
-
- // Apply filtering to repetition and definition levels
- if (current_filter_map->has_filter()) {
- if (current_filter_map->filter_all()) {
- _rep_levels.resize(0);
- _def_levels.resize(0);
- } else {
- std::vector<level_t> filtered_rep_levels;
- std::vector<level_t> filtered_def_levels;
- filtered_rep_levels.reserve(_rep_levels.size());
- filtered_def_levels.reserve(_def_levels.size());
-
- const uint8_t* filter_map_data = current_filter_map->filter_map_data();
-
- for (size_t i = 0; i < _rep_levels.size(); i++) {
- if (filter_map_data[i]) {
- filtered_rep_levels.push_back(_rep_levels[i]);
- filtered_def_levels.push_back(_def_levels[i]);
+ if (ancestor_null_indices.size() != 0) {
+ RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_null_indices.size(), false));
+ }
+ if (filter_map.has_filter()) {
+ auto new_rep_sz = before_rep_level_sz;
+ for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) {
+ if (nested_filter_map_data[idx - before_rep_level_sz]) {
+ _rep_levels[new_rep_sz] = _rep_levels[idx];
+ _def_levels[new_rep_sz] = _def_levels[idx];
+ new_rep_sz++;
}
}
-
- _rep_levels = std::move(filtered_rep_levels);
- _def_levels = std::move(filtered_def_levels);
+ _rep_levels.resize(new_rep_sz);
+ _def_levels.resize(new_rep_sz);
+ }
+ return Status::OK();
+ };
+
+ while (_current_range_idx < _row_ranges.range_size()) {
+ size_t left_row =
+ std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx));
+ size_t right_row = std::min(left_row + batch_size - *read_rows,
+ (size_t)_row_ranges.get_range_to(_current_range_idx));
+ _current_row_index = left_row;
+ RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row));
+ size_t load_rows = 0;
+ bool cross_page = false;
+ size_t before_rep_level_sz = _rep_levels.size();
+ RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row,
+ &load_rows, &cross_page));
+ RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index));
+ _filter_map_index += load_rows;
+ while (cross_page) {
+ before_rep_level_sz = _rep_levels.size();
+ RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page));
+ RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1));
+ }
+ *read_rows += load_rows;
+ _current_row_index += load_rows;
+ _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx));
+ if (*read_rows == batch_size) {
+ break;
}
}
-
- // Prepare for next row
- ++_orig_filter_map_index;
-
- if (_rep_levels.size() > 0) {
- // make sure the rows of complex type are aligned correctly,
- // so the repetition level of first element should be 0, meaning a new row is started.
- DCHECK_EQ(_rep_levels[0], 0);
- }
+ *eof = _current_range_idx == _row_ranges.range_size();
return Status::OK();
}
-Status ScalarColumnReader::read_dict_values_to_column(MutableColumnPtr& doris_column,
- bool* has_dict) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::read_dict_values_to_column(
+ MutableColumnPtr& doris_column, bool* has_dict) {
bool loaded;
RETURN_IF_ERROR(_try_load_dict_page(&loaded, has_dict));
if (loaded && *has_dict) {
@@ -599,28 +517,24 @@ Status ScalarColumnReader::read_dict_values_to_column(MutableColumnPtr& doris_co
}
return Status::OK();
}
-
-MutableColumnPtr ScalarColumnReader::convert_dict_column_to_string_column(
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+MutableColumnPtr
+ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::convert_dict_column_to_string_column(
const ColumnInt32* dict_column) {
return _chunk_reader->convert_dict_column_to_string_column(dict_column);
}
-Status ScalarColumnReader::_try_load_dict_page(bool* loaded, bool* has_dict) {
- *loaded = false;
- *has_dict = false;
- if (_chunk_reader->remaining_num_values() == 0) {
- if (!_chunk_reader->has_next_page()) {
- *loaded = false;
- return Status::OK();
- }
- RETURN_IF_ERROR(_chunk_reader->next_page());
- *loaded = true;
- *has_dict = _chunk_reader->has_dict();
- }
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::_try_load_dict_page(bool* loaded,
+ bool* has_dict) {
+ // _chunk_reader init will load first page header to check whether has dict page
+ *loaded = true;
+ *has_dict = _chunk_reader->has_dict();
return Status::OK();
}
-Status ScalarColumnReader::read_column_data(
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
@@ -642,33 +556,33 @@ Status ScalarColumnReader::read_column_data(
doris_column, type, is_dict_filter);
DataTypePtr& resolved_type = _converter->get_physical_type();
- do {
- if (_chunk_reader->remaining_num_values() == 0) {
- if (!_chunk_reader->has_next_page()) {
- *eof = true;
- *read_rows = 0;
- return Status::OK();
- }
- RETURN_IF_ERROR(_chunk_reader->next_page());
- }
- if (_nested_column) {
- RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
- RETURN_IF_ERROR(_read_nested_column(resolved_column, resolved_type, filter_map,
- batch_size, read_rows, eof, is_dict_filter, false));
- break;
- }
+ _def_levels.clear();
+ _rep_levels.clear();
+ *read_rows = 0;
+
+ if constexpr (IN_COLLECTION) {
+ RETURN_IF_ERROR(_read_nested_column(resolved_column, resolved_type, filter_map, batch_size,
+ read_rows, eof, is_dict_filter));
+ return _converter->convert(resolved_column, _field_schema->data_type, type, doris_column,
+ is_dict_filter);
+ }
+ int64_t right_row = 0;
+ if constexpr (OFFSET_INDEX == false) {
+ RETURN_IF_ERROR(_chunk_reader->parse_page_header());
+ right_row = _chunk_reader->page_end_row();
+ } else {
+ right_row = _chunk_reader->page_end_row();
+ }
+ auto before_filter_map_index = _filter_map_index;
+
+ do {
// generate the row ranges that should be read
RowRanges read_ranges;
- _generate_read_ranges(RowRange {_current_row_index,
- _current_row_index + _chunk_reader->remaining_num_values()},
- &read_ranges);
-
+ _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges);
if (read_ranges.count() == 0) {
// skip the whole page
- _current_row_index += _chunk_reader->remaining_num_values();
- RETURN_IF_ERROR(_chunk_reader->skip_page());
- *read_rows = 0;
+ _current_row_index = right_row;
} else {
bool skip_whole_batch = false;
// Determining whether to skip page or batch will increase the calculation time.
@@ -680,12 +594,8 @@ Status ScalarColumnReader::read_column_data(
filter_map.can_filter_all(remaining_num_values, _filter_map_index)) {
// We can skip the whole page if the remaining values are filtered by predicate columns
_filter_map_index += remaining_num_values;
- _current_row_index += _chunk_reader->remaining_num_values();
- RETURN_IF_ERROR(_chunk_reader->skip_page());
+ _current_row_index = right_row;
*read_rows = remaining_num_values;
- if (!_chunk_reader->has_next_page()) {
- *eof = true;
- }
break;
}
skip_whole_batch = batch_size <= remaining_num_values &&
@@ -695,6 +605,7 @@ Status ScalarColumnReader::read_column_data(
}
}
// load page data to decode or skip values
+ RETURN_IF_ERROR(_chunk_reader->parse_page_header());
RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent());
size_t has_read = 0;
for (size_t idx = 0; idx < read_ranges.range_size(); idx++) {
@@ -713,18 +624,34 @@ Status ScalarColumnReader::read_column_data(
filter_map, is_dict_filter));
}
has_read += read_values;
+ *read_rows += read_values;
_current_row_index += read_values;
if (has_read == batch_size) {
break;
}
}
- *read_rows = has_read;
}
+ } while (false);
- if (_chunk_reader->remaining_num_values() == 0 && !_chunk_reader->has_next_page()) {
+ if (right_row == _current_row_index) {
+ if (!_chunk_reader->has_next_page()) {
*eof = true;
+ } else {
+ RETURN_IF_ERROR(_chunk_reader->next_page());
}
- } while (false);
+ }
+
+ if (filter_map.has_filter()) {
+ size_t new_rep_sz = 0;
+ for (size_t idx = before_filter_map_index; idx < _filter_map_index; idx++) {
+ if (filter_map.filter_map_data()[idx]) {
+ _def_levels[new_rep_sz] = _def_levels[idx - before_filter_map_index];
+ new_rep_sz++;
+ }
+ }
+ _def_levels.resize(new_rep_sz);
+ }
+ _rep_levels.resize(_def_levels.size(), 0);
return _converter->convert(resolved_column, _field_schema->data_type, type, doris_column,
is_dict_filter);
@@ -955,7 +882,7 @@ Status StructColumnReader::read_column_data(
field_rows += loop_rows;
}
DCHECK_EQ(*read_rows, field_rows);
- DCHECK_EQ(*eof, field_eof);
+ // DCHECK_EQ(*eof, field_eof);
}
}
@@ -1076,6 +1003,12 @@ Status StructColumnReader::read_column_data(
}
return Status::OK();
}
+
+template class ScalarColumnReader<true, true>;
+template class ScalarColumnReader<true, false>;
+template class ScalarColumnReader<false, true>;
+template class ScalarColumnReader<false, false>;
+
#include "common/compile_check_end.h"
}; // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index c318d559b8a..98df81fb540 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -70,7 +70,7 @@ public:
parse_page_header_num(0),
read_page_header_time(0) {}
- Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReader::Statistics& cs,
+ Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReaderStatistics& cs,
int64_t null_map_time)
: read_time(fs.read_time),
read_calls(fs.read_calls),
@@ -120,9 +120,9 @@ public:
}
};
- ParquetColumnReader(const RowRanges& row_ranges, const cctz::time_zone* ctz,
+ ParquetColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz,
io::IOContext* io_ctx)
- : _row_ranges(row_ranges), _ctz(ctz), _io_ctx(io_ctx) {}
+ : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {}
virtual ~ParquetColumnReader() = default;
virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
@@ -143,10 +143,9 @@ public:
const tparquet::RowGroup& row_group, const RowRanges& row_ranges,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
std::unique_ptr<ParquetColumnReader>& reader, size_t max_buf_size,
- const tparquet::OffsetIndex* offset_index = nullptr,
- const std::set<uint64_t>& column_ids = {},
+ std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
+ bool in_collection = false, const std::set<uint64_t>& column_ids = {},
const std::set<uint64_t>& filter_column_ids = {});
- void set_nested_column() { _nested_column = true; }
virtual const std::vector<level_t>& get_rep_level() const = 0;
virtual const std::vector<level_t>& get_def_level() const = 0;
virtual Statistics statistics() = 0;
@@ -160,9 +159,8 @@ protected:
void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const;
FieldSchema* _field_schema = nullptr;
- // When scalar column is the child of nested column, we should turn off the filtering by page index and lazy read.
- bool _nested_column = false;
const RowRanges& _row_ranges;
+ size_t _total_rows = 0;
const cctz::time_zone* _ctz = nullptr;
io::IOContext* _io_ctx = nullptr;
int64_t _current_row_index = 0;
@@ -172,13 +170,15 @@ protected:
std::set<uint64_t> _filter_column_ids;
};
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
class ScalarColumnReader : public ParquetColumnReader {
ENABLE_FACTORY_CREATOR(ScalarColumnReader)
public:
- ScalarColumnReader(const RowRanges& row_ranges, const tparquet::ColumnChunk& chunk_meta,
+ ScalarColumnReader(const RowRanges& row_ranges, size_t total_rows,
+ const tparquet::ColumnChunk& chunk_meta,
const tparquet::OffsetIndex* offset_index, const cctz::time_zone* ctz,
io::IOContext* io_ctx)
- : ParquetColumnReader(row_ranges, ctz, io_ctx),
+ : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx),
_chunk_meta(chunk_meta),
_offset_index(offset_index) {}
~ScalarColumnReader() override { close(); }
@@ -204,11 +204,78 @@ public:
private:
tparquet::ColumnChunk _chunk_meta;
- const tparquet::OffsetIndex* _offset_index;
+ const tparquet::OffsetIndex* _offset_index = nullptr;
std::unique_ptr<io::BufferedFileStreamReader> _stream_reader;
- std::unique_ptr<ColumnChunkReader> _chunk_reader;
+ std::unique_ptr<ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>> _chunk_reader;
+ // rep def levels buffer.
std::vector<level_t> _rep_levels;
std::vector<level_t> _def_levels;
+
+ size_t _current_range_idx = 0;
+
+ Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx,
+ std::vector<uint16_t>& null_map,
+ std::unordered_set<size_t>& ancestor_null_indices) {
+ size_t has_read = level_start_idx;
+ null_map.emplace_back(0);
+ bool prev_is_null = false;
+
+ while (has_read < level_end_idx) {
+ level_t def_level = _def_levels[has_read++];
+ size_t loop_read = 1;
+ while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) {
+ has_read++;
+ loop_read++;
+ }
+
+ if (def_level < _field_schema->repeated_parent_def_level) {
+ for (size_t i = 0; i < loop_read; i++) {
+ ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i);
+ }
+ continue;
+ }
+
+ bool is_null = def_level < _field_schema->definition_level;
+
+ if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) {
+ null_map.back() += loop_read;
+ } else {
+ if (!(prev_is_null ^ is_null)) {
+ null_map.emplace_back(0);
+ }
+ size_t remaining = loop_read;
+ while (remaining > USHRT_MAX) {
+ null_map.emplace_back(USHRT_MAX);
+ null_map.emplace_back(0);
+ remaining -= USHRT_MAX;
+ }
+ null_map.emplace_back((u_short)remaining);
+ prev_is_null = is_null;
+ }
+ }
+ return Status::OK();
+ }
+
+ Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx,
+ size_t level_end_idx, std::vector<uint8_t>& nested_filter_map_data,
+ std::unique_ptr<FilterMap>* nested_filter_map) {
+ nested_filter_map_data.resize(level_end_idx - level_start_idx);
+ for (size_t idx = level_start_idx; idx < level_end_idx; idx++) {
+ if (idx != level_start_idx && _rep_levels[idx] == 0) {
+ filter_loc++;
+ }
+ nested_filter_map_data[idx - level_start_idx] =
+ filter_map.filter_map_data()[filter_loc];
+ }
+
+ auto new_filter = std::make_unique<FilterMap>();
+ RETURN_IF_ERROR(new_filter->init(nested_filter_map_data.data(),
+ nested_filter_map_data.size(), false));
+ *nested_filter_map = std::move(new_filter);
+
+ return Status::OK();
+ }
+
std::unique_ptr<parquet::PhysicalToLogicalConverter> _converter = nullptr;
std::unique_ptr<std::vector<uint8_t>> _nested_filter_map_data = nullptr;
size_t _orig_filter_map_index = 0;
@@ -217,17 +284,17 @@ private:
Status _read_values(size_t num_values, ColumnPtr& doris_column, DataTypePtr& type,
FilterMap& filter_map, bool is_dict_filter);
Status _read_nested_column(ColumnPtr& doris_column, DataTypePtr& type, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
- bool align_rows);
+ size_t batch_size, size_t* read_rows, bool* eof,
+ bool is_dict_filter);
Status _try_load_dict_page(bool* loaded, bool* has_dict);
};
class ArrayColumnReader : public ParquetColumnReader {
ENABLE_FACTORY_CREATOR(ArrayColumnReader)
public:
- ArrayColumnReader(const RowRanges& row_ranges, const cctz::time_zone* ctz,
+ ArrayColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz,
io::IOContext* io_ctx)
- : ParquetColumnReader(row_ranges, ctz, io_ctx) {}
+ : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {}
~ArrayColumnReader() override { close(); }
Status init(std::unique_ptr<ParquetColumnReader> element_reader, FieldSchema* field);
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
@@ -252,8 +319,9 @@ private:
class MapColumnReader : public ParquetColumnReader {
ENABLE_FACTORY_CREATOR(MapColumnReader)
public:
- MapColumnReader(const RowRanges& row_ranges, const cctz::time_zone* ctz, io::IOContext* io_ctx)
- : ParquetColumnReader(row_ranges, ctz, io_ctx) {}
+ MapColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz,
+ io::IOContext* io_ctx)
+ : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {}
~MapColumnReader() override { close(); }
Status init(std::unique_ptr<ParquetColumnReader> key_reader,
@@ -292,9 +360,9 @@ private:
class StructColumnReader : public ParquetColumnReader {
ENABLE_FACTORY_CREATOR(StructColumnReader)
public:
- StructColumnReader(const RowRanges& row_ranges, const cctz::time_zone* ctz,
+ StructColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz,
io::IOContext* io_ctx)
- : ParquetColumnReader(row_ranges, ctz, io_ctx) {}
+ : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {}
~StructColumnReader() override { close(); }
Status init(
@@ -357,9 +425,9 @@ private:
// This is used when a column is not needed but its structure is required (e.g., for map keys)
class SkipReadingReader : public ParquetColumnReader {
public:
- SkipReadingReader(const RowRanges& row_ranges, const cctz::time_zone* ctz,
+ SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz,
io::IOContext* io_ctx, FieldSchema* field_schema)
- : ParquetColumnReader(row_ranges, ctz, io_ctx) {
+ : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {
_field_schema = field_schema; // Use inherited member from base class
VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: "
<< _field_schema->name;
@@ -399,10 +467,11 @@ public:
}
static std::unique_ptr<SkipReadingReader> create_unique(const RowRanges& row_ranges,
- cctz::time_zone* ctz,
+ size_t total_rows, cctz::time_zone* ctz,
io::IOContext* io_ctx,
FieldSchema* field_schema) {
- return std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field_schema);
+ return std::make_unique<SkipReadingReader>(row_ranges, total_rows, ctz, io_ctx,
+ field_schema);
}
// These methods should not be called for SkipReadingReader
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index 58f1c337555..ad3a322e475 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -129,17 +129,11 @@ Status RowGroupReader::init(
std::min(MAX_COLUMN_BUF_SIZE, MAX_GROUP_BUF_SIZE / _read_table_columns.size());
for (const auto& read_table_col : _read_table_columns) {
auto read_file_col = _table_info_node_ptr->children_file_column_name(read_table_col);
-
auto* field = schema.get_column(read_file_col);
- auto physical_index = field->physical_column_index;
std::unique_ptr<ParquetColumnReader> reader;
- // TODO : support rested column types
- const tparquet::OffsetIndex* offset_index =
- col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index]
- : nullptr;
RETURN_IF_ERROR(ParquetColumnReader::create(
_file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader,
- max_buf_size, offset_index, _column_ids, _filter_column_ids));
+ max_buf_size, col_offsets, false, _column_ids, _filter_column_ids));
if (reader == nullptr) {
VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed";
return Status::Corruption("Init row group reader failed");
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
index 7d221e7d970..19d4edf6d26 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
@@ -40,23 +40,33 @@ namespace doris::vectorized {
#include "common/compile_check_begin.h"
static constexpr size_t INIT_PAGE_HEADER_SIZE = 128;
-std::unique_ptr<PageReader> create_page_reader(io::BufferedStreamReader* reader,
- io::IOContext* io_ctx, uint64_t offset,
- uint64_t length, int64_t num_values,
- const tparquet::OffsetIndex* offset_index) {
- if (offset_index) {
- return std::make_unique<PageReaderWithOffsetIndex>(reader, io_ctx, offset, length,
- num_values, offset_index);
- } else {
- return std::make_unique<PageReader>(reader, io_ctx, offset, length);
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+PageReader<IN_COLLECTION, OFFSET_INDEX>::PageReader(io::BufferedStreamReader* reader,
+ io::IOContext* io_ctx, uint64_t offset,
+ uint64_t length, size_t total_rows,
+ const tparquet::OffsetIndex* offset_index)
+ : _reader(reader),
+ _io_ctx(io_ctx),
+ _offset(offset),
+ _start_offset(offset),
+ _end_offset(offset + length),
+ _total_rows(total_rows),
+ _offset_index(offset_index) {
+ _next_header_offset = _offset;
+ _state = INITIALIZED;
+
+ if constexpr (OFFSET_INDEX) {
+ _end_row = _offset_index->page_locations.size() >= 2
+ ? _offset_index->page_locations[1].first_row_index
+ : _total_rows;
}
}
-PageReader::PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset,
- uint64_t length)
- : _reader(reader), _io_ctx(io_ctx), _start_offset(offset), _end_offset(offset + length) {}
-
-Status PageReader::_parse_page_header() {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status PageReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
+ if (_state == HEADER_PARSED) {
+ return Status::OK();
+ }
if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) {
return Status::IOError("Out-of-bounds Access");
}
@@ -97,6 +107,14 @@ Status PageReader::_parse_page_header() {
header_size <<= 2;
}
+ if constexpr (OFFSET_INDEX == false) {
+ if (is_header_v2()) {
+ _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows;
+ } else if constexpr (!IN_COLLECTION) {
+ _end_row = _start_row + _cur_page_header.data_page_header.num_values;
+ }
+ }
+
_statistics.parse_page_header_num++;
_offset += real_header_size;
_next_header_offset = _offset + _cur_page_header.compressed_page_size;
@@ -104,16 +122,8 @@ Status PageReader::_parse_page_header() {
return Status::OK();
}
-Status PageReader::skip_page() {
- if (UNLIKELY(_state != HEADER_PARSED)) {
- return Status::IOError("Should generate page header first to skip current page");
- }
- _offset = _next_header_offset;
- _state = INITIALIZED;
- return Status::OK();
-}
-
-Status PageReader::get_page_data(Slice& slice) {
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+Status PageReader<IN_COLLECTION, OFFSET_INDEX>::get_page_data(Slice& slice) {
if (UNLIKELY(_state != HEADER_PARSED)) {
return Status::IOError("Should generate page header first to load current page data");
}
@@ -123,9 +133,14 @@ Status PageReader::get_page_data(Slice& slice) {
slice.size = _cur_page_header.compressed_page_size;
RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx));
_offset += slice.size;
- _state = INITIALIZED;
+ _state = DATA_LOADED;
return Status::OK();
}
+
+template class PageReader<true, true>;
+template class PageReader<true, false>;
+template class PageReader<false, true>;
+template class PageReader<false, false>;
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.h b/be/src/vec/exec/format/parquet/vparquet_page_reader.h
index 17827163641..16dc49f030d 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.h
@@ -24,6 +24,17 @@
#include "common/cast_set.h"
#include "common/status.h"
+#include "util/block_compression.h"
+#include "vec/exec/format/parquet/parquet_common.h"
+namespace doris {
+class BlockCompressionCodec;
+
+namespace io {
+class BufferedStreamReader;
+struct IOContext;
+} // namespace io
+
+} // namespace doris
namespace doris {
namespace io {
@@ -38,6 +49,8 @@ namespace doris::vectorized {
/**
* Use to deserialize parquet page header, and get the page data in iterator interface.
*/
+
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
class PageReader {
public:
struct Statistics {
@@ -48,146 +61,118 @@ public:
};
PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset,
- uint64_t length);
- virtual ~PageReader() = default;
+ uint64_t length, size_t total_rows,
+ const tparquet::OffsetIndex* offset_index = nullptr);
+ ~PageReader() = default;
- // Deprecated
- // Parquet file may not be standardized,
- // _end_offset may exceed the actual data area.
- // ColumnChunkReader::has_next_page() use the number of parsed values for judgment
- // [[deprecated]]
- bool has_next_page() const { return _offset < _end_offset; }
+ bool has_next_page() const {
+ if constexpr (OFFSET_INDEX) {
+ return _page_index + 1 != _offset_index->page_locations.size();
+ } else {
+ // Deprecated
+ // Parquet file may not be standardized,
+ // _end_offset may exceed the actual data area.
+ // ColumnChunkReader::has_next_page() use the number of parsed values for judgment
+ // ref:https://github.com/duckdb/duckdb/issues/10829
+ // [[deprecated]]
+ LOG(FATAL) << "has_next_page should not be called when no offset index";
+ return _offset < _end_offset;
+ }
+ }
- virtual Status next_page_header() { return _parse_page_header(); }
+ Status parse_page_header();
- virtual Status get_page_header(const tparquet::PageHeader*& page_header) {
- if (UNLIKELY(_state != HEADER_PARSED)) {
- return Status::InternalError("Page header not parsed");
+ Status next_page() {
+ _statistics.skip_page_header_num += _state == INITIALIZED;
+ if constexpr (OFFSET_INDEX) {
+ _page_index++;
+ _start_row = _offset_index->page_locations[_page_index].first_row_index;
+ if (_page_index + 1 < _offset_index->page_locations.size()) {
+ _end_row = _offset_index->page_locations[_page_index + 1].first_row_index;
+ } else {
+ _end_row = _total_rows;
+ }
+ int64_t next_page_offset = _offset_index->page_locations[_page_index].offset;
+ _offset = next_page_offset;
+ _next_header_offset = next_page_offset;
+ _state = INITIALIZED;
+ } else {
+ if (UNLIKELY(_offset == _start_offset)) {
+ return Status::Corruption("should parse first page.");
+ }
+
+ if (is_header_v2()) {
+ _start_row += _cur_page_header.data_page_header_v2.num_rows;
+ } else if constexpr (!IN_COLLECTION) {
+ _start_row += _cur_page_header.data_page_header.num_values;
+ }
+
+ _offset = _next_header_offset;
+ _state = INITIALIZED;
}
- page_header = &_cur_page_header;
+
return Status::OK();
}
- virtual Status get_num_values(uint32_t& num_values) {
- if (_state != HEADER_PARSED) {
- return Status::InternalError("Page header not parsed");
- }
- if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) {
- num_values = _cur_page_header.data_page_header_v2.num_values;
+ Status dict_next_page() {
+ if constexpr (OFFSET_INDEX) {
+ _state = INITIALIZED;
+ return Status::OK();
} else {
- num_values = _cur_page_header.data_page_header.num_values;
+ return next_page();
}
- return Status::OK();
}
- virtual Status skip_page();
+ Status get_page_header(const tparquet::PageHeader*& page_header) {
+ if (UNLIKELY(_state != HEADER_PARSED)) {
+ return Status::InternalError("Page header not parsed");
+ }
+ page_header = &_cur_page_header;
+ return Status::OK();
+ }
- virtual Status get_page_data(Slice& slice);
+ Status get_page_data(Slice& slice);
Statistics& statistics() { return _statistics; }
- void seek_to_page(int64_t page_header_offset) {
- _offset = page_header_offset;
- _next_header_offset = page_header_offset;
- _state = INITIALIZED;
- }
+ bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; }
-protected:
- enum PageReaderState { INITIALIZED, HEADER_PARSED };
- PageReaderState _state = INITIALIZED;
- tparquet::PageHeader _cur_page_header;
- Statistics _statistics;
+ size_t start_row() const { return _start_row; }
- Status _parse_page_header();
+ size_t end_row() const { return _end_row; }
private:
+ enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED };
+ PageReaderState _state = INITIALIZED;
+ Statistics _statistics;
+
io::BufferedStreamReader* _reader = nullptr;
io::IOContext* _io_ctx = nullptr;
+ // current reader offset in file location.
uint64_t _offset = 0;
- uint64_t _next_header_offset = 0;
+ // this page offset in file location.
uint64_t _start_offset = 0;
uint64_t _end_offset = 0;
-};
-
-class PageReaderWithOffsetIndex : public PageReader {
-public:
- PageReaderWithOffsetIndex(io::BufferedStreamReader* reader, io::IOContext* io_ctx,
- uint64_t offset, uint64_t length, int64_t num_values,
- const tparquet::OffsetIndex* offset_index)
- : PageReader(reader, io_ctx, offset, length),
- _num_values(num_values),
- _offset_index(offset_index) {}
-
- Status next_page_header() override {
- // lazy to parse page header in get_page_header
- return Status::OK();
- }
-
- Status get_page_header(const tparquet::PageHeader*& page_header) override {
- if (_state != HEADER_PARSED) {
- RETURN_IF_ERROR(_parse_page_header());
- }
- page_header = &_cur_page_header;
- return Status::OK();
- }
-
- Status get_num_values(uint32_t& num_values) override {
- if (UNLIKELY(_page_index >= _offset_index->page_locations.size())) {
- return Status::IOError("End of page");
- }
-
- if (_page_index < _offset_index->page_locations.size() - 1) {
- num_values = cast_set<uint32_t>(
- _offset_index->page_locations[_page_index + 1].first_row_index -
- _offset_index->page_locations[_page_index].first_row_index);
- } else {
- num_values = cast_set<uint32_t>(
- _num_values - _offset_index->page_locations[_page_index].first_row_index);
- }
- return Status::OK();
- }
-
- Status skip_page() override {
- if (UNLIKELY(_page_index >= _offset_index->page_locations.size())) {
- return Status::IOError("End of page");
- }
-
- if (_state != HEADER_PARSED) {
- _statistics.skip_page_header_num++;
- }
-
- seek_to_page(_offset_index->page_locations[_page_index].offset +
- _offset_index->page_locations[_page_index].compressed_page_size);
- _page_index++;
- return Status::OK();
- }
-
- Status get_page_data(Slice& slice) override {
- if (_page_index >= _offset_index->page_locations.size()) {
- return Status::IOError("End of page");
- }
- if (_state != HEADER_PARSED) {
- RETURN_IF_ERROR(_parse_page_header());
- }
-
- // dirctionary page is not in page location
- if (LIKELY(_cur_page_header.type != tparquet::PageType::DICTIONARY_PAGE)) {
- _page_index++;
- }
-
- return PageReader::get_page_data(slice);
- }
-
-private:
+ uint64_t _next_header_offset = 0;
+ // current page row range
+ size_t _start_row = 0;
+ size_t _end_row = 0;
+ // total rows in this column chunk
+ size_t _total_rows = 0;
+ // for page index
size_t _page_index = 0;
- int64_t _num_values = 0;
const tparquet::OffsetIndex* _offset_index;
+
+ tparquet::PageHeader _cur_page_header;
};
-std::unique_ptr<PageReader> create_page_reader(io::BufferedStreamReader* reader,
- io::IOContext* io_ctx, uint64_t offset,
- uint64_t length, int64_t num_values = 0,
- const tparquet::OffsetIndex* offset_index = nullptr);
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+std::unique_ptr<PageReader<IN_COLLECTION, OFFSET_INDEX>> create_page_reader(
+ io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length,
+ size_t total_rows, const tparquet::OffsetIndex* offset_index = nullptr) {
+ return std::make_unique<PageReader<IN_COLLECTION, OFFSET_INDEX>>(reader, io_ctx, offset, length,
+ total_rows, offset_index);
+}
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index d4688621ca1..77d8212ab01 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -889,6 +889,41 @@ Status ParquetReader::_process_page_index_filter(
return Status::OK();
}
+ std::vector<int> parquet_col_ids;
+ for (size_t idx = 0; idx < _read_table_columns.size(); idx++) {
+ const auto& read_table_col = _read_table_columns[idx];
+ const auto& read_file_col = _read_file_columns[idx];
+ if (!_colname_to_slot_id->contains(read_table_col)) {
+ continue;
+ }
+ auto* field = _file_metadata->schema().get_column(read_file_col);
+
+ std::function<void(FieldSchema * field)> f = [&](FieldSchema* field) {
+ if (!_column_ids.empty() &&
+ _column_ids.find(field->get_column_id()) == _column_ids.end()) {
+ return;
+ }
+
+ if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
+ f(&field->children[0]);
+ } else if (field->data_type->get_primitive_type() == TYPE_MAP) {
+ f(&field->children[0]);
+ f(&field->children[1]);
+ } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
+ for (int i = 0; i < field->children.size(); ++i) {
+ f(&field->children[i]);
+ }
+ } else {
+ int parquet_col_id = field->physical_column_index;
+ if (parquet_col_id >= 0) {
+ parquet_col_ids.push_back(parquet_col_id);
+ }
+ }
+ };
+
+ f(field);
+ }
+
auto parse_offset_index = [&]() -> Status {
std::vector<uint8_t> off_index_buff(page_index._offset_index_size);
Slice res(off_index_buff.data(), page_index._offset_index_size);
@@ -901,20 +936,8 @@ Status ParquetReader::_process_page_index_filter(
_column_statistics.read_bytes += bytes_read;
_column_statistics.page_index_read_calls++;
_col_offsets.clear();
- for (size_t idx = 0; idx < _read_table_columns.size(); idx++) {
- const auto& read_table_col = _read_table_columns[idx];
- const auto& read_file_col = _read_file_columns[idx];
- if (!_colname_to_slot_id->contains(read_table_col)) {
- // equal delete may add column to read_table_col, but this column no slot_id.
- continue;
- }
- int parquet_col_id =
- _file_metadata->schema().get_column(read_file_col)->physical_column_index;
- if (parquet_col_id < 0) {
- // complex type, not support page index yet.
- continue;
- }
+ for (auto parquet_col_id : parquet_col_ids) {
auto& chunk = row_group.columns[parquet_col_id];
if (chunk.offset_index_length == 0) [[unlikely]] {
continue;
@@ -932,7 +955,7 @@ Status ParquetReader::_process_page_index_filter(
RETURN_IF_ERROR(parse_offset_index());
// Check if page index is needed for min-max filter.
- if (!_enable_filter_by_min_max || _lazy_read_ctx.has_complex_type || push_down_pred.empty()) {
+ if (!_enable_filter_by_min_max || push_down_pred.empty()) {
read_whole_row_group();
return Status::OK();
}
diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
index 4f4202e1143..f6da2a3f65b 100644
--- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
+++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
@@ -177,7 +177,7 @@ static int fill_nullable_column(ColumnPtr& doris_column, level_t* definitions, s
static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::ColumnChunk* column_chunk,
FieldSchema* field_schema, ColumnPtr& doris_column,
- DataTypePtr& data_type, level_t* definitions) {
+ DataTypePtr& data_type, level_t* definitions, size_t total_rows) {
tparquet::ColumnMetaData chunk_meta = column_chunk->meta_data;
size_t start_offset = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset
: chunk_meta.data_page_offset;
@@ -197,12 +197,12 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column
io::BufferedFileStreamReader stream_reader(file_reader, start_offset, chunk_size, 1024);
- ColumnChunkReader chunk_reader(&stream_reader, column_chunk, field_schema, nullptr, &ctz,
- nullptr);
+ ColumnChunkReader<false, false> chunk_reader(&stream_reader, column_chunk, field_schema,
+ nullptr, total_rows, nullptr);
// initialize chunk reader
static_cast<void>(chunk_reader.init());
// seek to next page header
- static_cast<void>(chunk_reader.next_page());
+ static_cast<void>(chunk_reader.parse_page_header());
// load page data into underlying container
static_cast<void>(chunk_reader.load_page_data());
int rows = chunk_reader.remaining_num_values();
@@ -210,7 +210,7 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column
if (field_schema->definition_level == 0) { // required field
std::fill(definitions, definitions + rows, 1);
} else {
- chunk_reader.get_def_levels(definitions, rows);
+ chunk_reader._def_level_decoder.get_levels(definitions, rows);
}
MutableColumnPtr data_column;
if (src_column->is_nullable()) {
@@ -240,10 +240,11 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column
if (definitions[i] != level_type) {
if (level_type == 0) {
// null values
- chunk_reader.insert_null_values(data_column, num_values);
+ data_column->insert_many_defaults(num_values);
} else {
std::vector<u_short> null_map = {(u_short)num_values};
- RETURN_IF_ERROR(run_length_map.init(null_map, rows, nullptr, &filter_map, 0));
+ RETURN_IF_ERROR(
+ run_length_map.init(null_map, num_values, nullptr, &filter_map, 0));
RETURN_IF_ERROR(chunk_reader.decode_values(data_column, resolved_type,
run_length_map, false));
}
@@ -255,10 +256,10 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column
}
if (level_type == 0) {
// null values
- chunk_reader.insert_null_values(data_column, num_values);
+ data_column->insert_many_defaults(num_values);
} else {
std::vector<u_short> null_map = {(u_short)num_values};
- RETURN_IF_ERROR(run_length_map.init(null_map, rows, nullptr, &filter_map, 0));
+ RETURN_IF_ERROR(run_length_map.init(null_map, num_values, nullptr, &filter_map, 0));
RETURN_IF_ERROR(
chunk_reader.decode_values(data_column, resolved_type, run_length_map, false));
}
@@ -414,7 +415,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file,
static_cast<void>(
get_column_values(reader, &t_metadata.row_groups[0].columns[c],
const_cast<FieldSchema*>(schema_descriptor.get_column(c)),
- data_column, data_type, defs.data()));
+ data_column, data_type, defs.data(), rows));
}
// `date_v2_col` date, // 14 - 13, DATEV2
{
@@ -424,7 +425,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file,
static_cast<void>(
get_column_values(reader, &t_metadata.row_groups[0].columns[13],
const_cast<FieldSchema*>(schema_descriptor.get_column(13)),
- data_column, data_type, defs.data()));
+ data_column, data_type, defs.data(), rows));
}
// `timestamp_v2_col` timestamp, // 15 - 9, DATETIMEV2
{
@@ -434,7 +435,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file,
static_cast<void>(
get_column_values(reader, &t_metadata.row_groups[0].columns[9],
const_cast<FieldSchema*>(schema_descriptor.get_column(9)),
- data_column, data_type, defs.data()));
+ data_column, data_type, defs.data(), rows));
}
io::FileReaderSPtr result;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql
index 7f374dacfc0..21f381cfa70 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql
@@ -21,3 +21,24 @@ CREATE TABLE `parquet_topn_lazy_mat_table`(
msck repair table orc_topn_lazy_mat_table;
msck repair table parquet_topn_lazy_mat_table;
+
+
+
+CREATE TABLE `parquet_topn_lazy_complex_table`(
+ id INT,
+ col1 STRING,
+ col2 STRUCT<a: INT, b: ARRAY<INT>>,
+ col3 MAP<INT, ARRAY<STRING>>
+) STORED AS PARQUET LOCATION
+ '/user/doris/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/';
+
+CREATE TABLE `parquet_topn_lazy_complex_table_multi_pages`(
+ id INT,
+ col1 STRING,
+ col2 STRUCT<a: INT, b: ARRAY<INT>>,
+ col3 MAP<INT, ARRAY<STRING>>
+) STORED AS PARQUET LOCATION
+ '/user/doris/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/';
+
+msck repair table parquet_topn_lazy_complex_table;
+msck repair table parquet_topn_lazy_complex_table_multi_pages;
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet
new file mode 100644
index 00000000000..7bac48aa2c1
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet
new file mode 100644
index 00000000000..fd8d8852078
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet
new file mode 100644
index 00000000000..881fc9e7304
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet
new file mode 100644
index 00000000000..ddde09317ba
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet
new file mode 100644
index 00000000000..1d9a3707dc2
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet
new file mode 100644
index 00000000000..307da5f60b0
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet
new file mode 100644
index 00000000000..1ea7d9e8461
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet
new file mode 100644
index 00000000000..4edcc8841ba
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet differ
diff --git a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
index 248bf4b90f7..0d4e28fd9f0 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out
@@ -1240,6 +1240,720 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+{"a":null, "b":[17, 18]} 17
+{"a":18, "b":null} 18
+{"a":19, "b":[1, null]} 19
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+{"a":null, "b":[17, 18]} 17
+{"a":18, "b":null} 18
+{"a":19, "b":[1, null]} 19
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+
-- !1 --
1 user1 1.0 false 0.5 1
2 user2 2.0 true 1.0 1
@@ -2481,3 +3195,716 @@
1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1
1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+{"a":null, "b":[17, 18]} 17
+{"a":18, "b":null} 18
+{"a":19, "b":[1, null]} 19
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_1 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_2 --
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_3 --
+text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_4 --
+{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]}
+{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]}
+{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]}
+{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null}
+{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]}
+{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]}
+{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null}
+{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]}
+
+-- !complex_5 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+
+-- !complex_6 --
+{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]}
+{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]}
+{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null}
+{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]}
+{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null}
+{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]}
+{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]}
+{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null}
+{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]}
+{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]}
+{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null}
+{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]}
+{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]}
+{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null}
+
+-- !complex_7 --
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+
+-- !complex_8 --
+3 text_3
+7 text_7
+12 text_12
+
+-- !complex_9 --
+8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8
+
+-- !complex_10 --
+{"a":1, "b":[1, 2]} 1
+{"a":10, "b":null} 10
+{"a":11, "b":[11, 12]} 11
+{"a":null, "b":[12, 13]} 12
+{"a":13, "b":null} 13
+{"a":14, "b":[14, 15]} 14
+{"a":null, "b":null} 15
+{"a":16, "b":[16, 17]} 16
+{"a":null, "b":[17, 18]} 17
+{"a":18, "b":null} 18
+{"a":19, "b":[1, null]} 19
+
+-- !complex_11 --
+2 text_2
+10 text_10
+20 text_20
+
+-- !complex_12 --
+2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null}
+4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]}
+6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]}
+8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null}
+10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]}
+12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null}
+14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]}
+16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]}
+18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]}
+20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]}
+22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]}
+24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null}
+
+-- !complex_13 --
+1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]}
+3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]}
+5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]}
+7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]}
+9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]}
+11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]}
+13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]}
+15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null}
+17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null}
+19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]}
+21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]}
+23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]}
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
index 6ec5438a518..7d6a3181cef 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
@@ -156,9 +156,28 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock
order by a.id,b.id limit 100;
"""
}
- }
-
+ for (String table : ["parquet_topn_lazy_complex_table", "parquet_topn_lazy_complex_table_multi_pages"]) {
+ for (int limit : limitValues) {
+ qt_complex_1 """ select * from ${table} order by id asc limit ${limit}; """
+ qt_complex_2 """ select * from ${table} order by col1 desc limit ${limit}; """
+ qt_complex_3 """ select col1,col3,col2 from ${table} order by id desc limit ${limit}; """
+ qt_complex_4 """ select col3,col2 from ${table} order by col1 desc limit ${limit}; """
+
+ qt_complex_5 """ select * from ${table} where id = 1 order by id limit ${limit}; """
+ qt_complex_6 """ select col3,col2 from ${table} where id > 10 order by id limit ${limit}; """
+ qt_complex_7 """ select * from ${table} where id between 5 and 15 order by id limit ${limit}; """
+ qt_complex_8 """ select id, col1 from ${table} where id in (3, 7, 12) order by id limit ${limit}; """
+
+ qt_complex_9 """ select id,col3,col1 from ${table} where col1 = 'text_8' order by id limit ${limit}; """
+ qt_complex_10 """ select col2,id from ${table} where col1 like 'text_1%' order by id limit ${limit}; """
+ qt_complex_11 """ select id, col1 from ${table} where col1 in ('text_2', 'text_10', 'text_20') order by id limit ${limit}; """
+
+ qt_complex_12 """ select * from ${table} where id%2 = 0 order by id limit ${limit}; """
+ qt_complex_13 """ select * from ${table} where id%2 = 1 order by id limit ${limit}; """
+ }
+ }
+ }
for (String hivePrefix : ["hive2"]) {
String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/03: [feature](vparquet-reader) Implements parquet file page cache.
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit db54cd036af850bb9173d9fb99fb9e7433ca2fb2
Author: kakachen <ch...@selectdb.com>
AuthorDate: Tue Dec 23 21:40:07 2025 +0800
[feature](vparquet-reader) Implements parquet file page cache.
---
be/src/common/config.cpp | 5 +
be/src/common/config.h | 4 +
be/src/io/cache/cached_remote_file_reader.h | 2 +
be/src/io/file_factory.cpp | 15 +
be/src/io/file_factory.h | 6 +
be/src/io/fs/broker_file_reader.cpp | 6 +-
be/src/io/fs/broker_file_reader.h | 5 +-
be/src/io/fs/broker_file_system.cpp | 2 +-
be/src/io/fs/buffered_reader.h | 12 +
be/src/io/fs/file_reader.h | 3 +
be/src/io/fs/hdfs_file_reader.cpp | 7 +-
be/src/io/fs/hdfs_file_reader.h | 5 +-
be/src/io/fs/http_file_reader.cpp | 7 +-
be/src/io/fs/http_file_reader.h | 5 +-
be/src/io/fs/http_file_system.cpp | 2 +-
be/src/io/fs/local_file_reader.h | 2 +
be/src/io/fs/s3_file_reader.h | 2 +
be/src/io/fs/stream_load_pipe.h | 2 +
be/src/io/fs/tracing_file_reader.h | 2 +
be/src/vec/exec/format/orc/orc_file_reader.h | 2 +
.../parquet/vparquet_column_chunk_reader.cpp | 313 ++++++--
.../format/parquet/vparquet_column_chunk_reader.h | 51 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 28 +-
.../exec/format/parquet/vparquet_column_reader.h | 45 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 13 +-
.../exec/format/parquet/vparquet_group_reader.h | 3 +-
.../exec/format/parquet/vparquet_page_reader.cpp | 141 +++-
.../vec/exec/format/parquet/vparquet_page_reader.h | 88 ++-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 44 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 8 +
be/test/io/fs/buffered_reader_test.cpp | 6 +
.../format/file_reader/file_meta_cache_test.cpp | 2 +
.../format/parquet/parquet_page_cache_test.cpp | 859 +++++++++++++++++++++
be/test/vec/exec/orc/orc_file_reader_test.cpp | 2 +
.../java/org/apache/doris/qe/SessionVariable.java | 27 +
gensrc/thrift/PaloInternalService.thrift | 4 +
36 files changed, 1636 insertions(+), 94 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 03a25a6f891..e9f5792028c 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -428,6 +428,11 @@ DEFINE_Int32(index_page_cache_percentage, "10");
DEFINE_mBool(disable_storage_page_cache, "false");
// whether to disable row cache feature in storage
DEFINE_mBool(disable_storage_row_cache, "true");
+// Parquet page cache: threshold ratio for caching decompressed vs compressed pages
+// If uncompressed_size / compressed_size <= threshold, cache decompressed; otherwise cache compressed
+DEFINE_Double(parquet_page_cache_decompress_threshold, "1.5");
+// Parquet page cache: whether to enable caching compressed pages (when ratio exceeds threshold)
+DEFINE_Bool(enable_parquet_cache_compressed_pages, "false");
// whether to disable pk page cache feature in storage
DEFINE_Bool(disable_pk_storage_page_cache, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 7b19e8e4e4d..09e397f6323 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -444,6 +444,10 @@ DECLARE_Int32(index_page_cache_percentage);
DECLARE_Bool(disable_storage_page_cache);
// whether to disable row cache feature in storage
DECLARE_mBool(disable_storage_row_cache);
+// Parquet page cache: threshold ratio for caching decompressed vs compressed pages
+DECLARE_Double(parquet_page_cache_decompress_threshold);
+// Parquet page cache: whether to enable caching compressed pages
+DECLARE_Bool(enable_parquet_cache_compressed_pages);
// whether to disable pk page cache feature in storage
DECLARE_Bool(disable_pk_storage_page_cache);
diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h
index 939471b62ea..20c1a47ce88 100644
--- a/be/src/io/cache/cached_remote_file_reader.h
+++ b/be/src/io/cache/cached_remote_file_reader.h
@@ -55,6 +55,8 @@ public:
static std::pair<size_t, size_t> s_align_size(size_t offset, size_t size, size_t length);
+ int64_t mtime() const override { return _remote_file_reader->mtime(); }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp
index bd08bc20461..07484935643 100644
--- a/be/src/io/file_factory.cpp
+++ b/be/src/io/file_factory.cpp
@@ -203,6 +203,21 @@ Result<io::FileReaderSPtr> FileFactory::create_file_reader(
const io::FileSystemProperties& system_properties,
const io::FileDescription& file_description, const io::FileReaderOptions& reader_options,
RuntimeProfile* profile) {
+ auto reader_res = _create_file_reader_internal(system_properties, file_description,
+ reader_options, profile);
+ if (!reader_res.has_value()) {
+ return unexpected(std::move(reader_res).error());
+ }
+ auto file_reader = std::move(reader_res).value();
+ LOG_INFO("create file reader for path={}, size={}, mtime={}", file_description.path,
+ file_description.file_size, file_description.mtime);
+ return file_reader;
+}
+
+Result<io::FileReaderSPtr> FileFactory::_create_file_reader_internal(
+ const io::FileSystemProperties& system_properties,
+ const io::FileDescription& file_description, const io::FileReaderOptions& reader_options,
+ RuntimeProfile* profile) {
TFileType::type type = system_properties.system_type;
switch (type) {
case TFileType::FILE_LOCAL: {
diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h
index 0ba791bd0a3..61e322ca0af 100644
--- a/be/src/io/file_factory.h
+++ b/be/src/io/file_factory.h
@@ -126,6 +126,12 @@ public:
private:
static std::string _get_fs_name(const io::FileDescription& file_description);
+
+ /// Create FileReader without FS
+ static Result<io::FileReaderSPtr> _create_file_reader_internal(
+ const io::FileSystemProperties& system_properties,
+ const io::FileDescription& file_description,
+ const io::FileReaderOptions& reader_options, RuntimeProfile* profile = nullptr);
};
} // namespace doris
diff --git a/be/src/io/fs/broker_file_reader.cpp b/be/src/io/fs/broker_file_reader.cpp
index 102ea3e2477..41b2992f700 100644
--- a/be/src/io/fs/broker_file_reader.cpp
+++ b/be/src/io/fs/broker_file_reader.cpp
@@ -39,12 +39,14 @@ struct IOContext;
BrokerFileReader::BrokerFileReader(const TNetworkAddress& broker_addr, Path path, size_t file_size,
TBrokerFD fd,
- std::shared_ptr<BrokerServiceConnection> connection)
+ std::shared_ptr<BrokerServiceConnection> connection,
+ int64_t mtime)
: _path(std::move(path)),
_file_size(file_size),
_broker_addr(broker_addr),
_fd(fd),
- _connection(std::move(connection)) {
+ _connection(std::move(connection)),
+ _mtime(mtime) {
DorisMetrics::instance()->broker_file_open_reading->increment(1);
DorisMetrics::instance()->broker_file_reader_total->increment(1);
}
diff --git a/be/src/io/fs/broker_file_reader.h b/be/src/io/fs/broker_file_reader.h
index 7d19edb32c0..2f6bd94b652 100644
--- a/be/src/io/fs/broker_file_reader.h
+++ b/be/src/io/fs/broker_file_reader.h
@@ -38,7 +38,7 @@ struct IOContext;
class BrokerFileReader final : public FileReader {
public:
BrokerFileReader(const TNetworkAddress& broker_addr, Path path, size_t file_size, TBrokerFD fd,
- std::shared_ptr<BrokerServiceConnection> connection);
+ std::shared_ptr<BrokerServiceConnection> connection, int64_t mtime = 0);
~BrokerFileReader() override;
@@ -50,6 +50,8 @@ public:
bool closed() const override { return _closed.load(std::memory_order_acquire); }
+ int64_t mtime() const override { return _mtime; }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
@@ -62,6 +64,7 @@ private:
TBrokerFD _fd;
std::shared_ptr<BrokerServiceConnection> _connection;
+ int64_t _mtime;
std::atomic<bool> _closed = false;
};
} // namespace doris::io
diff --git a/be/src/io/fs/broker_file_system.cpp b/be/src/io/fs/broker_file_system.cpp
index 8b0d5db23e2..b0dc89dc277 100644
--- a/be/src/io/fs/broker_file_system.cpp
+++ b/be/src/io/fs/broker_file_system.cpp
@@ -139,7 +139,7 @@ Status BrokerFileSystem::open_file_internal(const Path& file, FileReaderSPtr* re
error_msg(response->opStatus.message));
}
*reader = std::make_shared<BrokerFileReader>(_broker_addr, file, fsize, response->fd,
- _connection);
+ _connection, opts.mtime);
return Status::OK();
}
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index 5fe07176235..6ddcca02067 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -160,6 +160,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return _inner_reader->mtime(); }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
@@ -329,6 +331,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return _reader->mtime(); }
+
// for test only
size_t buffer_remaining() const { return _remaining; }
@@ -532,6 +536,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return _reader->mtime(); }
+
void set_random_access_ranges(const std::vector<PrefetchRange>* random_access_ranges) {
_random_access_ranges = random_access_ranges;
for (auto& _pre_buffer : _pre_buffers) {
@@ -592,6 +598,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return _reader->mtime(); }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
@@ -626,6 +634,8 @@ public:
virtual ~BufferedStreamReader() = default;
// return the file path
virtual std::string path() = 0;
+
+ virtual int64_t mtime() const = 0;
};
class BufferedFileStreamReader : public BufferedStreamReader, public ProfileCollector {
@@ -639,6 +649,8 @@ public:
Status read_bytes(Slice& slice, uint64_t offset, const IOContext* io_ctx) override;
std::string path() override { return _file->path(); }
+ int64_t mtime() const override { return _file->mtime(); }
+
protected:
void _collect_profile_before_close() override {
if (_file != nullptr) {
diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h
index e6d8527e831..3df912cbad4 100644
--- a/be/src/io/fs/file_reader.h
+++ b/be/src/io/fs/file_reader.h
@@ -90,6 +90,9 @@ public:
virtual const std::string& get_data_dir_path() { return VIRTUAL_REMOTE_DATA_DIR; }
+ // File modification time (seconds since epoch). Default to 0 meaning unknown.
+ virtual int64_t mtime() const = 0;
+
protected:
virtual Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) = 0;
diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp
index 0e278dff0c8..b1d65a63ba0 100644
--- a/be/src/io/fs/hdfs_file_reader.cpp
+++ b/be/src/io/fs/hdfs_file_reader.cpp
@@ -66,16 +66,17 @@ Result<FileReaderSPtr> HdfsFileReader::create(Path full_path, const hdfsFS& fs,
auto path = convert_path(full_path, fs_name);
return get_file(fs, path, opts.mtime, opts.file_size).transform([&](auto&& accessor) {
return std::make_shared<HdfsFileReader>(std::move(path), std::move(fs_name),
- std::move(accessor), profile);
+ std::move(accessor), profile, opts.mtime);
});
}
HdfsFileReader::HdfsFileReader(Path path, std::string fs_name, FileHandleCache::Accessor accessor,
- RuntimeProfile* profile)
+ RuntimeProfile* profile, int64_t mtime)
: _path(std::move(path)),
_fs_name(std::move(fs_name)),
_accessor(std::move(accessor)),
- _profile(profile) {
+ _profile(profile),
+ _mtime(mtime) {
_handle = _accessor.get();
DorisMetrics::instance()->hdfs_file_open_reading->increment(1);
diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h
index 8556eea0de6..08f98bca29a 100644
--- a/be/src/io/fs/hdfs_file_reader.h
+++ b/be/src/io/fs/hdfs_file_reader.h
@@ -45,7 +45,7 @@ public:
const FileReaderOptions& opts, RuntimeProfile* profile);
HdfsFileReader(Path path, std::string fs_name, FileHandleCache::Accessor accessor,
- RuntimeProfile* profile);
+ RuntimeProfile* profile, int64_t mtime = 0);
~HdfsFileReader() override;
@@ -57,6 +57,8 @@ public:
bool closed() const override { return _closed.load(std::memory_order_acquire); }
+ int64_t mtime() const override { return _mtime; }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
@@ -86,6 +88,7 @@ private:
CachedHdfsFileHandle* _handle = nullptr; // owned by _cached_file_handle
std::atomic<bool> _closed = false;
RuntimeProfile* _profile = nullptr;
+ int64_t _mtime;
#ifdef USE_HADOOP_HDFS
HDFSProfile _hdfs_profile;
#endif
diff --git a/be/src/io/fs/http_file_reader.cpp b/be/src/io/fs/http_file_reader.cpp
index fb243179baf..5ad984039fc 100644
--- a/be/src/io/fs/http_file_reader.cpp
+++ b/be/src/io/fs/http_file_reader.cpp
@@ -34,7 +34,7 @@ Result<FileReaderSPtr> HttpFileReader::create(const std::string& url,
ofi.path = Path(url);
ofi.extend_info = props;
- auto reader = std::make_shared<HttpFileReader>(ofi, url);
+ auto reader = std::make_shared<HttpFileReader>(ofi, url, opts.mtime);
// Open the file to detect Range support and validate configuration
RETURN_IF_ERROR_RESULT(reader->open(opts));
@@ -42,11 +42,12 @@ Result<FileReaderSPtr> HttpFileReader::create(const std::string& url,
return reader;
}
-HttpFileReader::HttpFileReader(const OpenFileInfo& fileInfo, std::string url)
+HttpFileReader::HttpFileReader(const OpenFileInfo& fileInfo, std::string url, int64_t mtime)
: _extend_kv(fileInfo.extend_info),
_path(fileInfo.path),
_url(std::move(url)),
- _client(std::make_unique<HttpClient>()) {
+ _client(std::make_unique<HttpClient>()),
+ _mtime(mtime) {
auto etag_iter = _extend_kv.find("etag");
if (etag_iter != _extend_kv.end()) {
_etag = etag_iter->second;
diff --git a/be/src/io/fs/http_file_reader.h b/be/src/io/fs/http_file_reader.h
index 607eedf3d1a..982e65905aa 100644
--- a/be/src/io/fs/http_file_reader.h
+++ b/be/src/io/fs/http_file_reader.h
@@ -41,7 +41,7 @@ public:
const std::map<std::string, std::string>& props,
const FileReaderOptions& opts, RuntimeProfile* profile);
- explicit HttpFileReader(const OpenFileInfo& fileInfo, std::string url);
+ explicit HttpFileReader(const OpenFileInfo& fileInfo, std::string url, int64_t mtime);
~HttpFileReader() override;
Status open(const FileReaderOptions& opts);
@@ -52,6 +52,8 @@ public:
bool closed() const override { return _closed.load(std::memory_order_acquire); }
size_t size() const override { return _file_size; }
+ int64_t mtime() const override { return _mtime; }
+
private:
// Prepare and initialize the HTTP client for a new request
Status prepare_client(bool set_fail_on_error = true);
@@ -78,6 +80,7 @@ private:
int64_t _last_modified = 0;
std::atomic<bool> _closed = false;
std::unique_ptr<HttpClient> _client;
+ int64_t _mtime;
// Configuration for non-Range request handling
bool _enable_range_request = true; // Whether Range request is required
diff --git a/be/src/io/fs/http_file_system.cpp b/be/src/io/fs/http_file_system.cpp
index 92e175ca774..b1e8de354ad 100644
--- a/be/src/io/fs/http_file_system.cpp
+++ b/be/src/io/fs/http_file_system.cpp
@@ -56,7 +56,7 @@ Status HttpFileSystem::open_file_internal(const Path& path, FileReaderSPtr* read
// Pass properties (including HTTP headers) to the file reader
file_info.extend_info = _properties;
- auto http_reader = std::make_shared<HttpFileReader>(file_info, path.native());
+ auto http_reader = std::make_shared<HttpFileReader>(file_info, path.native(), opts.mtime);
RETURN_IF_ERROR(http_reader->open(opts));
*reader = http_reader;
return Status::OK();
diff --git a/be/src/io/fs/local_file_reader.h b/be/src/io/fs/local_file_reader.h
index 0ffd6ccde9e..97e071226a7 100644
--- a/be/src/io/fs/local_file_reader.h
+++ b/be/src/io/fs/local_file_reader.h
@@ -62,6 +62,8 @@ public:
const std::string& get_data_dir_path() override { return _data_dir_path; }
+ int64_t mtime() const override { return 0; }
+
private:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h
index 58294ec1891..40e3ac61d3a 100644
--- a/be/src/io/fs/s3_file_reader.h
+++ b/be/src/io/fs/s3_file_reader.h
@@ -53,6 +53,8 @@ public:
bool closed() const override { return _closed.load(std::memory_order_acquire); }
+ int64_t mtime() const override { return 0; }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
diff --git a/be/src/io/fs/stream_load_pipe.h b/be/src/io/fs/stream_load_pipe.h
index cedab0b6c17..df137a9267c 100644
--- a/be/src/io/fs/stream_load_pipe.h
+++ b/be/src/io/fs/stream_load_pipe.h
@@ -57,6 +57,8 @@ public:
size_t size() const override { return 0; }
+ int64_t mtime() const override { return 0; }
+
// called when consumer finished
Status close() override {
if (!(_finished || _cancelled)) {
diff --git a/be/src/io/fs/tracing_file_reader.h b/be/src/io/fs/tracing_file_reader.h
index 39b70dfbb63..7a6651afd21 100644
--- a/be/src/io/fs/tracing_file_reader.h
+++ b/be/src/io/fs/tracing_file_reader.h
@@ -47,6 +47,8 @@ public:
void _collect_profile_at_runtime() override { return _inner->collect_profile_at_runtime(); }
void _collect_profile_before_close() override { return _inner->collect_profile_before_close(); }
+ int64_t mtime() const override { return _inner->mtime(); }
+
FileReaderStats* stats() const { return _stats; }
doris::io::FileReaderSPtr inner_reader() { return _inner; }
diff --git a/be/src/vec/exec/format/orc/orc_file_reader.h b/be/src/vec/exec/format/orc/orc_file_reader.h
index 503777e67c2..15aeed33242 100644
--- a/be/src/vec/exec/format/orc/orc_file_reader.h
+++ b/be/src/vec/exec/format/orc/orc_file_reader.h
@@ -54,6 +54,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return _inner_reader->mtime(); }
+
// for test only
const Statistics& statistics() const { return _statistics; }
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
index a474b76a518..807975def2f 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
@@ -25,6 +25,8 @@
#include <utility>
#include "common/compiler_util.h" // IWYU pragma: keep
+#include "io/fs/buffered_reader.h"
+#include "olap/page_cache.h"
#include "util/bit_util.h"
#include "util/block_compression.h"
#include "util/runtime_profile.h"
@@ -51,7 +53,7 @@ template <bool IN_COLLECTION, bool OFFSET_INDEX>
ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::ColumnChunkReader(
io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk,
FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows,
- io::IOContext* io_ctx)
+ io::IOContext* io_ctx, const ParquetPageReadContext& ctx)
: _field_schema(field_schema),
_max_rep_level(field_schema->repetition_level),
_max_def_level(field_schema->definition_level),
@@ -59,7 +61,8 @@ ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::ColumnChunkReader(
_metadata(column_chunk->meta_data),
_offset_index(offset_index),
_total_rows(total_rows),
- _io_ctx(io_ctx) {}
+ _io_ctx(io_ctx),
+ _ctx(ctx) {}
template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::init() {
@@ -68,7 +71,8 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::init() {
size_t chunk_size = _metadata.total_compressed_size;
// create page reader
_page_reader = create_page_reader<IN_COLLECTION, OFFSET_INDEX>(
- _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _offset_index);
+ _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata,
+ _offset_index, _ctx);
// get the block compression codec
RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec));
_state = INITIALIZED;
@@ -103,7 +107,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_parse_first_page_header(
RETURN_IF_ERROR(parse_page_header());
const tparquet::PageHeader* header = nullptr;
- RETURN_IF_ERROR(_page_reader->get_page_header(header));
+ RETURN_IF_ERROR(_page_reader->get_page_header(&header));
if (header->type == tparquet::PageType::DICTIONARY_PAGE) {
// the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false,
// so we should parse the directory page in next_page()
@@ -124,8 +128,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
RETURN_IF_ERROR(_page_reader->parse_page_header());
const tparquet::PageHeader* header = nullptr;
- ;
- RETURN_IF_ERROR(_page_reader->get_page_header(header));
+ RETURN_IF_ERROR(_page_reader->get_page_header(&header));
int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values
: header->data_page_header.num_values;
_remaining_rep_nums = page_num_values;
@@ -168,37 +171,151 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
}
const tparquet::PageHeader* header = nullptr;
- RETURN_IF_ERROR(_page_reader->get_page_header(header));
+ RETURN_IF_ERROR(_page_reader->get_page_header(&header));
int32_t uncompressed_size = header->uncompressed_page_size;
+ bool page_loaded = false;
+
+ // First, try to reuse a cache handle previously discovered by PageReader
+ // (header-only lookup) to avoid a second lookup here. If no handle is
+ // attached, fall back to a StoragePageCache lookup for a decompressed page.
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr) {
+ if (_page_reader->has_page_cache_handle()) {
+ const PageCacheHandle& handle = _page_reader->page_cache_handle();
+ Slice cached = handle.data();
+ size_t header_size = _page_reader->header_bytes().size();
+ //size_t levels_size = 0;
+ size_t levels_size = 0;
+ if (header->__isset.data_page_header_v2) {
+ const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2;
+ size_t rl = header_v2.repetition_levels_byte_length;
+ size_t dl = header_v2.definition_levels_byte_length;
+ levels_size = rl + dl;
+ _v2_rep_levels =
+ Slice(reinterpret_cast<const uint8_t*>(cached.data) + header_size, rl);
+ _v2_def_levels =
+ Slice(reinterpret_cast<const uint8_t*>(cached.data) + header_size + rl, dl);
+ }
+ // payload_slice points to the bytes after header and levels
+ Slice payload_slice(cached.data + header_size + levels_size,
+ cached.size - header_size - levels_size);
- if (_block_compress_codec != nullptr) {
- Slice compressed_data;
- RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data));
- if (header->__isset.data_page_header_v2) {
- const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2;
- // uncompressed_size = rl + dl + uncompressed_data_size
- // compressed_size = rl + dl + compressed_data_size
- uncompressed_size -= header_v2.repetition_levels_byte_length +
- header_v2.definition_levels_byte_length;
- _get_uncompressed_levels(header_v2, compressed_data);
+ bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed();
+
+ if (cache_payload_is_decompressed) {
+ // Cached payload is already uncompressed
+ _page_data = payload_slice;
+ } else {
+ CHECK(_block_compress_codec);
+ // Decompress cached payload into _decompress_buf for decoding
+ size_t uncompressed_payload_size =
+ header->__isset.data_page_header_v2
+ ? static_cast<size_t>(header->uncompressed_page_size) - levels_size
+ : static_cast<size_t>(header->uncompressed_page_size);
+ _reserve_decompress_buf(uncompressed_payload_size);
+ _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time);
+ _chunk_statistics.decompress_cnt++;
+ RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data));
+ }
+ // page cache counters were incremented when PageReader did the header-only
+ // cache lookup. Do not increment again to avoid double-counting.
+ page_loaded = true;
}
- bool is_v2_compressed =
- header->__isset.data_page_header_v2 && header->data_page_header_v2.is_compressed;
- if (header->__isset.data_page_header || is_v2_compressed) {
- // check decompressed buffer size
- _reserve_decompress_buf(uncompressed_size);
- _page_data = Slice(_decompress_buf.get(), uncompressed_size);
- SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time);
- _chunk_statistics.decompress_cnt++;
- RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data));
+ }
+
+ if (!page_loaded) {
+ if (_block_compress_codec != nullptr) {
+ Slice compressed_data;
+ RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data));
+ std::vector<uint8_t> level_bytes;
+ if (header->__isset.data_page_header_v2) {
+ const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2;
+ // uncompressed_size = rl + dl + uncompressed_data_size
+ // compressed_size = rl + dl + compressed_data_size
+ uncompressed_size -= header_v2.repetition_levels_byte_length +
+ header_v2.definition_levels_byte_length;
+ // copy level bytes (rl + dl) so that we can cache header + levels + uncompressed payload
+ size_t rl = header_v2.repetition_levels_byte_length;
+ size_t dl = header_v2.definition_levels_byte_length;
+ size_t level_sz = rl + dl;
+ if (level_sz > 0) {
+ level_bytes.resize(level_sz);
+ memcpy(level_bytes.data(), compressed_data.data, level_sz);
+ }
+ // now remove levels from compressed_data for decompression
+ _get_uncompressed_levels(header_v2, compressed_data);
+ }
+ bool is_v2_compressed = header->__isset.data_page_header_v2 &&
+ header->data_page_header_v2.is_compressed;
+ bool page_has_compression = header->__isset.data_page_header || is_v2_compressed;
+
+ if (page_has_compression) {
+ // Decompress payload for immediate decoding
+ _reserve_decompress_buf(uncompressed_size);
+ _page_data = Slice(_decompress_buf.get(), uncompressed_size);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time);
+ _chunk_statistics.decompress_cnt++;
+ RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data));
+
+ // Decide whether to cache decompressed payload or compressed payload based on threshold
+ bool should_cache_decompressed = false;
+ if (header->compressed_page_size > 0) {
+ should_cache_decompressed =
+ (_metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) ||
+ (static_cast<double>(header->uncompressed_page_size) <=
+ static_cast<double>(config::parquet_page_cache_decompress_threshold) *
+ static_cast<double>(header->compressed_page_size));
+ }
+
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr &&
+ !_page_reader->header_bytes().empty()) {
+ if (should_cache_decompressed) {
+ _insert_page_into_cache(level_bytes, _page_data);
+ _chunk_statistics.page_cache_decompressed_write_counter += 1;
+ } else {
+ if (config::enable_parquet_cache_compressed_pages) {
+ // cache the compressed payload as-is (header | levels | compressed_payload)
+ _insert_page_into_cache(
+ level_bytes, Slice(compressed_data.data, compressed_data.size));
+ _chunk_statistics.page_cache_compressed_write_counter += 1;
+ }
+ }
+ }
+ } else {
+ // no compression on this page, use the data directly
+ _page_data = Slice(compressed_data.data, compressed_data.size);
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr) {
+ _insert_page_into_cache(level_bytes, _page_data);
+ _chunk_statistics.page_cache_decompressed_write_counter += 1;
+ }
+ }
} else {
- // Don't need decompress
- _page_data = Slice(compressed_data.data, compressed_data.size);
- }
- } else {
- RETURN_IF_ERROR(_page_reader->get_page_data(_page_data));
- if (header->__isset.data_page_header_v2) {
- _get_uncompressed_levels(header->data_page_header_v2, _page_data);
+ // For uncompressed page, we may still need to extract v2 levels
+ std::vector<uint8_t> level_bytes;
+ Slice uncompressed_data;
+ RETURN_IF_ERROR(_page_reader->get_page_data(uncompressed_data));
+ if (header->__isset.data_page_header_v2) {
+ const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2;
+ size_t rl = header_v2.repetition_levels_byte_length;
+ size_t dl = header_v2.definition_levels_byte_length;
+ size_t level_sz = rl + dl;
+ if (level_sz > 0) {
+ level_bytes.resize(level_sz);
+ memcpy(level_bytes.data(), uncompressed_data.data, level_sz);
+ }
+ _get_uncompressed_levels(header_v2, uncompressed_data);
+ }
+ // copy page data out
+ _page_data = Slice(uncompressed_data.data, uncompressed_data.size);
+ // Optionally cache uncompressed data for uncompressed pages
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr) {
+ _insert_page_into_cache(level_bytes, _page_data);
+ _chunk_statistics.page_cache_decompressed_write_counter += 1;
+ }
}
}
@@ -243,7 +360,15 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
_decoders[static_cast<int>(encoding)] = std::move(page_decoder);
_page_decoder = _decoders[static_cast<int>(encoding)].get();
}
- // Reset page data for each page
+ // Reset page data for each page.
+ // If this is a v2 data page, _page_data currently contains rl+dl followed by payload.
+ // The decoder expects payload-only, so strip the level bytes into a temporary Slice
+ // that points into the same cached memory (so ownership remains with the cache handle).
+ // Slice payload_slice = _page_data;
+ // if (header->__isset.data_page_header_v2) {
+ // const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2;
+ // _get_uncompressed_levels(header_v2, payload_slice);
+ // }
RETURN_IF_ERROR(_page_decoder->set_data(&_page_data));
_state = DATA_LOADED;
@@ -253,7 +378,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_decode_dict_page() {
const tparquet::PageHeader* header = nullptr;
- RETURN_IF_ERROR(_page_reader->get_page_header(header));
+ RETURN_IF_ERROR(_page_reader->get_page_header(&header));
DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type);
SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time);
@@ -270,16 +395,88 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_decode_dict_page() {
// Prepare dictionary data
int32_t uncompressed_size = header->uncompressed_page_size;
auto dict_data = make_unique_buffer<uint8_t>(uncompressed_size);
- if (_block_compress_codec != nullptr) {
- Slice compressed_data;
- RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data));
- Slice dict_slice(dict_data.get(), uncompressed_size);
- RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &dict_slice));
- } else {
- Slice dict_slice;
- RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice));
- // The data is stored by BufferedStreamReader, we should copy it out
- memcpy(dict_data.get(), dict_slice.data, dict_slice.size);
+ bool dict_loaded = false;
+
+ // Try to load dictionary page from cache
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr) {
+ if (_page_reader->has_page_cache_handle()) {
+ const PageCacheHandle& handle = _page_reader->page_cache_handle();
+ Slice cached = handle.data();
+ size_t header_size = _page_reader->header_bytes().size();
+ // Dictionary page layout in cache: header | payload (compressed or uncompressed)
+ Slice payload_slice(cached.data + header_size, cached.size - header_size);
+
+ bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed();
+
+ if (cache_payload_is_decompressed) {
+ // Use cached decompressed dictionary data
+ memcpy(dict_data.get(), payload_slice.data, payload_slice.size);
+ dict_loaded = true;
+ } else {
+ CHECK(_block_compress_codec);
+ // Decompress cached compressed dictionary data
+ Slice dict_slice(dict_data.get(), uncompressed_size);
+ RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice));
+ dict_loaded = true;
+ }
+
+ // When dictionary page is loaded from cache, we need to skip the page data
+ // to update the offset correctly (similar to calling get_page_data())
+ if (dict_loaded) {
+ _page_reader->skip_page_data();
+ }
+ }
+ }
+
+ if (!dict_loaded) {
+ // Load and decompress dictionary page from file
+ if (_block_compress_codec != nullptr) {
+ Slice compressed_data;
+ RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data));
+ Slice dict_slice(dict_data.get(), uncompressed_size);
+ RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &dict_slice));
+
+ // Decide whether to cache decompressed or compressed dictionary based on threshold
+ bool should_cache_decompressed = false;
+ if (header->compressed_page_size > 0) {
+ should_cache_decompressed =
+ (static_cast<double>(header->uncompressed_page_size) <=
+ static_cast<double>(config::parquet_page_cache_decompress_threshold) *
+ static_cast<double>(header->compressed_page_size));
+ }
+
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr && !_page_reader->header_bytes().empty()) {
+ std::vector<uint8_t> empty_levels; // Dictionary pages don't have levels
+ if (should_cache_decompressed) {
+ // Cache the decompressed dictionary page
+ _insert_page_into_cache(empty_levels, dict_slice);
+ _chunk_statistics.page_cache_decompressed_write_counter += 1;
+ } else {
+ if (config::enable_parquet_cache_compressed_pages) {
+ // Cache the compressed dictionary page
+ _insert_page_into_cache(empty_levels,
+ Slice(compressed_data.data, compressed_data.size));
+ _chunk_statistics.page_cache_compressed_write_counter += 1;
+ }
+ }
+ }
+ } else {
+ Slice dict_slice;
+ RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice));
+ // The data is stored by BufferedStreamReader, we should copy it out
+ memcpy(dict_data.get(), dict_slice.data, dict_slice.size);
+
+ // Cache the uncompressed dictionary page
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr && !_page_reader->header_bytes().empty()) {
+ std::vector<uint8_t> empty_levels;
+ Slice payload(dict_data.get(), uncompressed_size);
+ _insert_page_into_cache(empty_levels, payload);
+ _chunk_statistics.page_cache_decompressed_write_counter += 1;
+ }
+ }
}
// Cache page decoder
@@ -305,6 +502,32 @@ void ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_reserve_decompress_buf(siz
}
}
+template <bool IN_COLLECTION, bool OFFSET_INDEX>
+void ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_insert_page_into_cache(
+ const std::vector<uint8_t>& level_bytes, const Slice& payload) {
+ StoragePageCache::CacheKey key(
+ fmt::format("{}::{}", _stream_reader->path(), _stream_reader->mtime()),
+ _page_reader->file_end_offset(), _page_reader->header_start_offset());
+ const std::vector<uint8_t>& header_bytes = _page_reader->header_bytes();
+ size_t total = header_bytes.size() + level_bytes.size() + payload.size;
+ auto* page = new DataPage(total, true, segment_v2::DATA_PAGE);
+ size_t pos = 0;
+ memcpy(page->data() + pos, header_bytes.data(), header_bytes.size());
+ pos += header_bytes.size();
+ if (!level_bytes.empty()) {
+ memcpy(page->data() + pos, level_bytes.data(), level_bytes.size());
+ pos += level_bytes.size();
+ }
+ if (payload.size > 0) {
+ memcpy(page->data() + pos, payload.data, payload.size);
+ pos += payload.size;
+ }
+ page->reset_size(total);
+ PageCacheHandle handle;
+ StoragePageCache::instance()->insert(key, page, &handle, segment_v2::DATA_PAGE);
+ _chunk_statistics.page_cache_write_counter += 1;
+}
+
template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::skip_values(size_t num_values,
bool skip_data) {
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
index 93d0f92ecec..c380e99e087 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
@@ -61,6 +61,18 @@ struct ColumnChunkReaderStatistics {
int64_t skip_page_header_num = 0;
int64_t parse_page_header_num = 0;
int64_t read_page_header_time = 0;
+ // page cache metrics
+ // total pages read (from cache or file)
+ int64_t page_read_counter = 0;
+ int64_t page_cache_write_counter = 0;
+ int64_t page_cache_compressed_write_counter = 0;
+ int64_t page_cache_decompressed_write_counter = 0;
+ // number of cache hits (either compressed or decompressed)
+ int64_t page_cache_hit_counter = 0;
+ int64_t page_cache_missing_counter = 0;
+ // per-hit breakdown
+ int64_t page_cache_compressed_hit_counter = 0;
+ int64_t page_cache_decompressed_hit_counter = 0;
};
/**
@@ -74,6 +86,17 @@ struct ColumnChunkReaderStatistics {
int64_t decode_level_time = 0;
int64_t skip_page_header_num = 0;
int64_t parse_page_header_num = 0;
+ // page cache metrics
+ // total pages read (from cache or file)
+ int64_t page_read_counter = 0;
+ int64_t page_cache_write_counter = 0;
+ int64_t page_cache_compressed_write_counter = 0;
+ int64_t page_cache_decompressed_write_counter = 0;
+ // number of cache hits (either compressed or decompressed)
+ int64_t page_cache_hit_counter = 0;
+ // per-hit breakdown
+ int64_t page_cache_compressed_hit_counter = 0;
+ int64_t page_cache_decompressed_hit_counter = 0;
};
* // Create chunk reader
* ColumnChunkReader chunk_reader(BufferedStreamReader* reader,
@@ -109,7 +132,8 @@ public:
ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk,
FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index,
- size_t total_row, io::IOContext* io_ctx);
+ size_t total_row, io::IOContext* io_ctx,
+ const ParquetPageReadContext& ctx = ParquetPageReadContext());
~ColumnChunkReader() = default;
// Initialize chunk reader, will generate the decoder and codec.
@@ -167,6 +191,21 @@ public:
_page_reader->page_statistics().parse_page_header_num;
_chunk_statistics.read_page_header_time =
_page_reader->page_statistics().read_page_header_time;
+ _chunk_statistics.page_read_counter += _page_reader->page_statistics().page_read_counter;
+ _chunk_statistics.page_cache_write_counter +=
+ _page_reader->page_statistics().page_cache_write_counter;
+ _chunk_statistics.page_cache_compressed_write_counter +=
+ _page_reader->page_statistics().page_cache_compressed_write_counter;
+ _chunk_statistics.page_cache_decompressed_write_counter +=
+ _page_reader->page_statistics().page_cache_decompressed_write_counter;
+ _chunk_statistics.page_cache_hit_counter +=
+ _page_reader->page_statistics().page_cache_hit_counter;
+ _chunk_statistics.page_cache_missing_counter +=
+ _page_reader->page_statistics().page_cache_missing_counter;
+ _chunk_statistics.page_cache_compressed_hit_counter +=
+ _page_reader->page_statistics().page_cache_compressed_hit_counter;
+ _chunk_statistics.page_cache_decompressed_hit_counter +=
+ _page_reader->page_statistics().page_cache_decompressed_hit_counter;
return _chunk_statistics;
}
@@ -205,6 +244,12 @@ public:
size_t* result_rows, bool* cross_page);
Status load_cross_page_nested_row(std::vector<level_t>& rep_levels, bool* cross_page);
+ // Test helpers / accessors
+ Slice get_page_data() const { return _page_data; }
+ const Slice& v2_rep_levels() const { return _v2_rep_levels; }
+ const Slice& v2_def_levels() const { return _v2_def_levels; }
+ ColumnChunkReaderStatistics& statistics() { return chunk_statistics(); }
+
private:
enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED };
@@ -214,6 +259,7 @@ private:
void _reserve_decompress_buf(size_t size);
int32_t _get_type_length();
+ void _insert_page_into_cache(const std::vector<uint8_t>& level_bytes, const Slice& payload);
void _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data);
Status _skip_nested_rows_in_page(size_t num_rows);
@@ -233,6 +279,9 @@ private:
std::unique_ptr<PageReader<IN_COLLECTION, OFFSET_INDEX>> _page_reader;
BlockCompressionCodec* _block_compress_codec = nullptr;
+ // Session-level parquet page cache options
+ ParquetPageReadContext _ctx;
+
LevelDecoder _rep_level_decoder;
LevelDecoder _def_level_decoder;
size_t _chunk_parsed_values = 0;
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index db6042ce703..12c6573ea74 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -111,13 +111,14 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
size_t max_buf_size,
std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
bool in_collection, const std::set<uint64_t>& column_ids,
- const std::set<uint64_t>& filter_column_ids) {
+ const std::set<uint64_t>& filter_column_ids, RuntimeState* state,
+ const std::string& created_by) {
size_t total_rows = row_group.num_rows;
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
element_reader, max_buf_size, col_offsets, true, column_ids,
- filter_column_ids));
+ filter_column_ids, state, created_by));
// element_reader->set_nested_column();
auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx);
RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field));
@@ -132,7 +133,7 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
// Create key reader
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
key_reader, max_buf_size, col_offsets, true, column_ids,
- filter_column_ids));
+ filter_column_ids, state, created_by));
// key_reader->set_nested_column();
} else {
auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, total_rows, ctz,
@@ -145,7 +146,7 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
// Create value reader
RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx,
value_reader, max_buf_size, col_offsets, true, column_ids,
- filter_column_ids));
+ filter_column_ids, state, created_by));
// value_reader->set_nested_column();
} else {
auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, total_rows, ctz,
@@ -208,14 +209,14 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
auto scalar_reader = ScalarColumnReader<true, false>::create_unique(
row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
- RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, created_by));
scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
} else {
auto scalar_reader = ScalarColumnReader<true, true>::create_unique(
row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
- RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, created_by));
scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
}
@@ -224,14 +225,14 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
auto scalar_reader = ScalarColumnReader<false, false>::create_unique(
row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
- RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, created_by));
scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
} else {
auto scalar_reader = ScalarColumnReader<false, true>::create_unique(
row_ranges, total_rows, chunk, offset_index, ctz, io_ctx);
- RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, created_by));
scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
}
@@ -249,7 +250,9 @@ void ParquetColumnReader::_generate_read_ranges(RowRange page_row_range,
template <bool IN_COLLECTION, bool OFFSET_INDEX>
Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::init(io::FileReaderSPtr file,
FieldSchema* field,
- size_t max_buf_size) {
+ size_t max_buf_size,
+ RuntimeState* state,
+ const std::string& created_by) {
_field_schema = field;
auto& chunk_meta = _chunk_meta.meta_data;
int64_t chunk_start = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset
@@ -265,8 +268,13 @@ Status ScalarColumnReader<IN_COLLECTION, OFFSET_INDEX>::init(io::FileReaderSPtr
}
_stream_reader = std::make_unique<io::BufferedFileStreamReader>(file, chunk_start, chunk_len,
prefetch_buffer_size);
+ // Build Parquet page read context: enable_parquet_file_page_cache from session, others from BE config
+ ParquetPageReadContext ctx(
+ (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache,
+ created_by);
+
_chunk_reader = std::make_unique<ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>>(
- _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx);
+ _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx);
RETURN_IF_ERROR(_chunk_reader->init());
return Status::OK();
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index 4a49473a69f..8bcd9712716 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -65,7 +65,15 @@ public:
decode_null_map_time(0),
skip_page_header_num(0),
parse_page_header_num(0),
- read_page_header_time(0) {}
+ read_page_header_time(0),
+ page_read_counter(0),
+ page_cache_write_counter(0),
+ page_cache_compressed_write_counter(0),
+ page_cache_decompressed_write_counter(0),
+ page_cache_hit_counter(0),
+ page_cache_missing_counter(0),
+ page_cache_compressed_hit_counter(0),
+ page_cache_decompressed_hit_counter(0) {}
ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time)
: page_index_read_calls(0),
@@ -78,7 +86,15 @@ public:
decode_null_map_time(null_map_time),
skip_page_header_num(cs.skip_page_header_num),
parse_page_header_num(cs.parse_page_header_num),
- read_page_header_time(cs.read_page_header_time) {}
+ read_page_header_time(cs.read_page_header_time),
+ page_read_counter(cs.page_read_counter),
+ page_cache_write_counter(cs.page_cache_write_counter),
+ page_cache_compressed_write_counter(cs.page_cache_compressed_write_counter),
+ page_cache_decompressed_write_counter(cs.page_cache_decompressed_write_counter),
+ page_cache_hit_counter(cs.page_cache_hit_counter),
+ page_cache_missing_counter(cs.page_cache_missing_counter),
+ page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter),
+ page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter) {}
int64_t page_index_read_calls;
int64_t decompress_time;
@@ -91,6 +107,14 @@ public:
int64_t skip_page_header_num;
int64_t parse_page_header_num;
int64_t read_page_header_time;
+ int64_t page_read_counter;
+ int64_t page_cache_write_counter;
+ int64_t page_cache_compressed_write_counter;
+ int64_t page_cache_decompressed_write_counter;
+ int64_t page_cache_hit_counter;
+ int64_t page_cache_missing_counter;
+ int64_t page_cache_compressed_hit_counter;
+ int64_t page_cache_decompressed_hit_counter;
void merge(ColumnStatistics& col_statistics) {
page_index_read_calls += col_statistics.page_index_read_calls;
@@ -104,6 +128,17 @@ public:
skip_page_header_num += col_statistics.skip_page_header_num;
parse_page_header_num += col_statistics.parse_page_header_num;
read_page_header_time += col_statistics.read_page_header_time;
+ page_read_counter += col_statistics.page_read_counter;
+ page_cache_write_counter += col_statistics.page_cache_write_counter;
+ page_cache_compressed_write_counter +=
+ col_statistics.page_cache_compressed_write_counter;
+ page_cache_decompressed_write_counter +=
+ col_statistics.page_cache_decompressed_write_counter;
+ page_cache_hit_counter += col_statistics.page_cache_hit_counter;
+ page_cache_missing_counter += col_statistics.page_cache_missing_counter;
+ page_cache_compressed_hit_counter += col_statistics.page_cache_compressed_hit_counter;
+ page_cache_decompressed_hit_counter +=
+ col_statistics.page_cache_decompressed_hit_counter;
}
};
@@ -132,7 +167,8 @@ public:
std::unique_ptr<ParquetColumnReader>& reader, size_t max_buf_size,
std::unordered_map<int, tparquet::OffsetIndex>& col_offsets,
bool in_collection = false, const std::set<uint64_t>& column_ids = {},
- const std::set<uint64_t>& filter_column_ids = {});
+ const std::set<uint64_t>& filter_column_ids = {},
+ RuntimeState* state = nullptr, const std::string& created_by = "");
virtual const std::vector<level_t>& get_rep_level() const = 0;
virtual const std::vector<level_t>& get_def_level() const = 0;
virtual ColumnStatistics column_statistics() = 0;
@@ -169,7 +205,8 @@ public:
_chunk_meta(chunk_meta),
_offset_index(offset_index) {}
~ScalarColumnReader() override { close(); }
- Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size);
+ Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size,
+ RuntimeState* state, const std::string& created_by = "");
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index 31161f2c6ed..d57264c1987 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -85,7 +85,8 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
const PositionDeleteContext& position_delete_ctx,
const LazyReadContext& lazy_read_ctx, RuntimeState* state,
const std::set<uint64_t>& column_ids,
- const std::set<uint64_t>& filter_column_ids)
+ const std::set<uint64_t>& filter_column_ids,
+ const std::string& created_by)
: _file_reader(file_reader),
_read_table_columns(read_columns),
_row_group_id(row_group_id),
@@ -98,7 +99,8 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
_state(state),
_obj_pool(new ObjectPool()),
_column_ids(column_ids),
- _filter_column_ids(filter_column_ids) {}
+ _filter_column_ids(filter_column_ids),
+ _created_by(created_by) {}
RowGroupReader::~RowGroupReader() {
_column_readers.clear();
@@ -131,9 +133,10 @@ Status RowGroupReader::init(
auto read_file_col = _table_info_node_ptr->children_file_column_name(read_table_col);
auto* field = schema.get_column(read_file_col);
std::unique_ptr<ParquetColumnReader> reader;
- RETURN_IF_ERROR(ParquetColumnReader::create(
- _file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader,
- max_buf_size, col_offsets, false, _column_ids, _filter_column_ids));
+ RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field, _row_group_meta,
+ _read_ranges, _ctz, _io_ctx, reader,
+ max_buf_size, col_offsets, false, _column_ids,
+ _filter_column_ids, _state, _created_by));
if (reader == nullptr) {
VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed";
return Status::Corruption("Init row group reader failed");
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
index e9c4470af17..f504292c035 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
@@ -159,7 +159,7 @@ public:
const PositionDeleteContext& position_delete_ctx,
const LazyReadContext& lazy_read_ctx, RuntimeState* state,
const std::set<uint64_t>& column_ids,
- const std::set<uint64_t>& filter_column_ids);
+ const std::set<uint64_t>& filter_column_ids, const std::string& created_by = "");
~RowGroupReader();
Status init(const FieldDescriptor& schema, RowRanges& row_ranges,
@@ -261,6 +261,7 @@ private:
std::shared_ptr<ObjectPool> _obj_pool;
const std::set<uint64_t>& _column_ids;
const std::set<uint64_t>& _filter_column_ids;
+ std::string _created_by; // Parquet file's created_by field
bool _is_row_group_filtered = false;
RowGroupIndex _current_row_group_idx {0, 0, 0};
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
index 3b6d7fdcb9b..38478ab2863 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
@@ -26,6 +26,8 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/config.h"
#include "io/fs/buffered_reader.h"
+#include "olap/page_cache.h"
+#include "parquet_common.h"
#include "util/runtime_profile.h"
#include "util/slice.h"
#include "util/thrift_util.h"
@@ -40,18 +42,57 @@ namespace doris::vectorized {
#include "common/compile_check_begin.h"
static constexpr size_t INIT_PAGE_HEADER_SIZE = 128;
+// // Check if the file was created by a version that always marks pages as compressed
+// // regardless of the actual compression state (parquet-cpp < 2.0.0)
+// static bool _is_always_compressed(const ParquetPageReadContext& ctx) {
+// if (ctx.created_by.empty()) {
+// return false;
+// }
+
+// // Parse the version string
+// std::unique_ptr<ParsedVersion> parsed_version;
+// Status status = VersionParser::parse(ctx.created_by, &parsed_version);
+// if (!status.ok()) {
+// return false;
+// }
+
+// // Check if it's parquet-cpp
+// if (parsed_version->application() != "parquet-cpp") {
+// return false;
+// }
+
+// // Check if version < 2.0.0
+// if (!parsed_version->version().has_value()) {
+// return false;
+// }
+
+// std::unique_ptr<SemanticVersion> semantic_version;
+// status = SemanticVersion::parse(parsed_version->version().value(), &semantic_version);
+// if (!status.ok()) {
+// return false;
+// }
+
+// // parquet-cpp versions < 2.0.0 always report compressed
+// static const SemanticVersion PARQUET_CPP_FIXED_VERSION(2, 0, 0);
+// return semantic_version->compare_to(PARQUET_CPP_FIXED_VERSION) < 0;
+// }
+
template <bool IN_COLLECTION, bool OFFSET_INDEX>
PageReader<IN_COLLECTION, OFFSET_INDEX>::PageReader(io::BufferedStreamReader* reader,
io::IOContext* io_ctx, uint64_t offset,
uint64_t length, size_t total_rows,
- const tparquet::OffsetIndex* offset_index)
+ const tparquet::ColumnMetaData& metadata,
+ const tparquet::OffsetIndex* offset_index,
+ const ParquetPageReadContext& ctx)
: _reader(reader),
_io_ctx(io_ctx),
_offset(offset),
_start_offset(offset),
_end_offset(offset + length),
_total_rows(total_rows),
- _offset_index(offset_index) {
+ _metadata(metadata),
+ _offset_index(offset_index),
+ _ctx(ctx) {
_next_header_offset = _offset;
_state = INITIALIZED;
@@ -77,11 +118,104 @@ Status PageReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
return Status::IOError("Should skip or load current page to get next page");
}
+ _page_statistics.page_read_counter += 1;
+
+ // Parse page header from file; header bytes are saved for possible cache insertion
const uint8_t* page_header_buf = nullptr;
size_t max_size = _end_offset - _offset;
size_t header_size = std::min(INIT_PAGE_HEADER_SIZE, max_size);
const size_t MAX_PAGE_HEADER_SIZE = config::parquet_header_max_size_mb << 20;
uint32_t real_header_size = 0;
+
+ // Try a header-only lookup in the page cache. Cached pages store
+ // header + optional v2 levels + uncompressed payload, so we can
+ // parse the page header directly from the cached bytes and avoid
+ // a file read for the header.
+ if (_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache &&
+ StoragePageCache::instance() != nullptr) {
+ PageCacheHandle handle;
+ StoragePageCache::CacheKey key(fmt::format("{}::{}", _reader->path(), _reader->mtime()),
+ _end_offset, _offset);
+ if (StoragePageCache::instance()->lookup(key, &handle, segment_v2::DATA_PAGE)) {
+ // Parse header directly from cached data
+ _page_cache_handle = std::move(handle);
+ Slice s = _page_cache_handle.data();
+ real_header_size = cast_set<uint32_t>(s.size);
+ SCOPED_RAW_TIMER(&_page_statistics.decode_header_time);
+ auto st = deserialize_thrift_msg(reinterpret_cast<const uint8_t*>(s.data),
+ &real_header_size, true, &_cur_page_header);
+ if (!st.ok()) return st;
+ // Increment page cache counters for a true cache hit on header+payload
+ _page_statistics.page_cache_hit_counter += 1;
+ // Detect whether the cached payload is compressed or decompressed and record
+ // the appropriate counter. Cached layout is: header | optional levels | payload
+
+ // Determine if payload is compressed using V2 is_compressed field if available
+ // bool payload_is_compressed = true;
+ // if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) {
+ // const auto& page_header_v2 = _cur_page_header.data_page_header_v2;
+ // if (page_header_v2.__isset.is_compressed) {
+ // payload_is_compressed = page_header_v2.is_compressed;
+ // }
+ // }
+
+ // // ARROW-17100: [C++][Parquet] Fix backwards compatibility for ParquetV2 data pages written prior to 3.0.0 per ARROW-10353 #13665
+ // // https://github.com/apache/arrow/pull/13665/files
+ // // Prior to parquet-cpp version 2.0.0, is_compressed was always set to false in column headers,
+ // // even if compression was used. See ARROW-17100.
+ // bool always_compressed = _is_always_compressed(_ctx);
+ // payload_is_compressed |= always_compressed;
+
+ // // Apply codec check: if codec is UNCOMPRESSED, payload cannot be compressed
+ // payload_is_compressed = payload_is_compressed &&
+ // (_metadata.codec != tparquet::CompressionCodec::UNCOMPRESSED);
+
+ // // Save the computed result for use by ColumnChunkReader
+ // _payload_is_compressed = payload_is_compressed;
+
+ bool is_cache_payload_decompressed = true;
+ if (_cur_page_header.compressed_page_size > 0) {
+ is_cache_payload_decompressed =
+ (_metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) ||
+ (static_cast<double>(_cur_page_header.uncompressed_page_size) <=
+ static_cast<double>(config::parquet_page_cache_decompress_threshold) *
+ static_cast<double>(_cur_page_header.compressed_page_size));
+ }
+
+ if (is_cache_payload_decompressed) {
+ _page_statistics.page_cache_decompressed_hit_counter += 1;
+ } else {
+ _page_statistics.page_cache_compressed_hit_counter += 1;
+ }
+
+ _is_cache_payload_decompressed = is_cache_payload_decompressed;
+
+ if constexpr (OFFSET_INDEX == false) {
+ if (is_header_v2()) {
+ _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows;
+ } else if constexpr (!IN_COLLECTION) {
+ _end_row = _start_row + _cur_page_header.data_page_header.num_values;
+ }
+ }
+
+ // Save header bytes for later use (e.g., to insert updated cache entries)
+ _header_buf.assign(s.data, s.data + real_header_size);
+ _last_header_size = real_header_size;
+ _page_statistics.parse_page_header_num++;
+ _offset += real_header_size;
+ _next_header_offset = _offset + _cur_page_header.compressed_page_size;
+ _state = HEADER_PARSED;
+ return Status::OK();
+ } else {
+ _page_statistics.page_cache_missing_counter += 1;
+ // Clear any existing cache handle on miss to avoid holding stale handle
+ _page_cache_handle = PageCacheHandle();
+ }
+ }
+ // NOTE: page cache lookup for *decompressed* page data is handled in
+ // ColumnChunkReader::load_page_data(). PageReader should only be
+ // responsible for parsing the header bytes from the file and saving
+ // them in `_header_buf` for possible later insertion into the cache.
while (true) {
if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) {
return Status::EndOfFile("stop");
@@ -115,6 +249,9 @@ Status PageReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
}
}
+ // Save header bytes for possible cache insertion later
+ _header_buf.assign(page_header_buf, page_header_buf + real_header_size);
+ _last_header_size = real_header_size;
_page_statistics.parse_page_header_num++;
_offset += real_header_size;
_next_header_offset = _offset + _cur_page_header.compressed_page_size;
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.h b/be/src/vec/exec/format/parquet/vparquet_page_reader.h
index 9246819d59c..a722507c742 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.h
@@ -23,7 +23,9 @@
#include <memory>
#include "common/cast_set.h"
+#include "common/config.h"
#include "common/status.h"
+#include "olap/page_cache.h"
#include "util/block_compression.h"
#include "vec/exec/format/parquet/parquet_common.h"
namespace doris {
@@ -50,6 +52,15 @@ namespace doris::vectorized {
* Use to deserialize parquet page header, and get the page data in iterator interface.
*/
+// Session-level options for parquet page reading/caching.
+struct ParquetPageReadContext {
+ bool enable_parquet_file_page_cache = true;
+ std::string created_by; // Parquet file's created_by field for version checking
+ ParquetPageReadContext() = default;
+ ParquetPageReadContext(bool enable, const std::string& created_by_str = "")
+ : enable_parquet_file_page_cache(enable), created_by(created_by_str) {}
+};
+
template <bool IN_COLLECTION, bool OFFSET_INDEX>
class PageReader {
public:
@@ -58,11 +69,29 @@ public:
int64_t skip_page_header_num = 0;
int64_t parse_page_header_num = 0;
int64_t read_page_header_time = 0;
+ // page cache metrics
+ // page_cache_hit_counter: number of times a cached page was used
+ int64_t page_cache_hit_counter = 0;
+ // page_cache_missing_counter: number of times a cached page was not found
+ int64_t page_cache_missing_counter = 0;
+ // page_cache_compressed_hit_counter: number of cache hits where the cached payload is compressed
+ int64_t page_cache_compressed_hit_counter = 0;
+ // page_cache_decompressed_hit_counter: number of cache hits where the cached payload is decompressed
+ int64_t page_cache_decompressed_hit_counter = 0;
+ // page_cache_write_counter: number of times this reader wrote an entry into the cache
+ int64_t page_cache_write_counter = 0;
+ // page_cache_compressed_write_counter: number of times this reader wrote compressed page into the cache
+ int64_t page_cache_compressed_write_counter = 0;
+ // page_cache_decompressed_write_counter: number of times this reader wrote decompressed page into the cache
+ int64_t page_cache_decompressed_write_counter = 0;
+ // page_read_counter: total pages read by this PageReader (includes cache hits and file reads)
+ int64_t page_read_counter = 0;
};
PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset,
- uint64_t length, size_t total_rows,
- const tparquet::OffsetIndex* offset_index = nullptr);
+ uint64_t length, size_t total_rows, const tparquet::ColumnMetaData& metadata,
+ const tparquet::OffsetIndex* offset_index = nullptr,
+ const ParquetPageReadContext& ctx = ParquetPageReadContext());
~PageReader() = default;
bool has_next_page() const {
@@ -123,24 +152,61 @@ public:
}
}
- Status get_page_header(const tparquet::PageHeader*& page_header) {
+ Status get_page_header(const tparquet::PageHeader** page_header) {
if (UNLIKELY(_state != HEADER_PARSED)) {
return Status::InternalError("Page header not parsed");
}
- page_header = &_cur_page_header;
+ *page_header = &_cur_page_header;
return Status::OK();
}
Status get_page_data(Slice& slice);
+ // Skip page data and update offset (used when data is loaded from cache)
+ void skip_page_data() {
+ if (_state == HEADER_PARSED) {
+ _offset += _cur_page_header.compressed_page_size;
+ _state = DATA_LOADED;
+ }
+ }
+
+ // Expose header bytes info for cache insertion
+ uint32_t last_header_size() const { return _last_header_size; }
+ const std::vector<uint8_t>& header_bytes() const { return _header_buf; }
+ // header start offset for current page
+ int64_t header_start_offset() const {
+ return static_cast<int64_t>(_next_header_offset) - static_cast<int64_t>(_last_header_size) -
+ static_cast<int64_t>(_cur_page_header.compressed_page_size);
+ }
+ uint64_t file_end_offset() const { return _end_offset; }
+ bool cached_decompressed() const {
+ return static_cast<double>(_cur_page_header.uncompressed_page_size) <=
+ static_cast<double>(config::parquet_page_cache_decompress_threshold) *
+ static_cast<double>(_cur_page_header.compressed_page_size);
+ }
+
PageStatistics& page_statistics() { return _page_statistics; }
bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; }
+ // Returns whether the current page's cache payload is decompressed
+ bool is_cache_payload_decompressed() const { return _is_cache_payload_decompressed; }
+
size_t start_row() const { return _start_row; }
size_t end_row() const { return _end_row; }
+ // Accessors for cache handle
+ bool has_page_cache_handle() const { return _page_cache_handle.cache() != nullptr; }
+ const doris::PageCacheHandle& page_cache_handle() const { return _page_cache_handle; }
+
+ // Page cache members
+ doris::PageCacheHandle _page_cache_handle;
+ // stored header bytes when cache miss so we can insert header+payload into cache
+ std::vector<uint8_t> _header_buf;
+ // last parsed header size in bytes
+ uint32_t _last_header_size = 0;
+
private:
enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED };
PageReaderState _state = INITIALIZED;
@@ -159,19 +225,27 @@ private:
size_t _end_row = 0;
// total rows in this column chunk
size_t _total_rows = 0;
+ // Column metadata for this column chunk
+ const tparquet::ColumnMetaData& _metadata;
// for page index
size_t _page_index = 0;
const tparquet::OffsetIndex* _offset_index;
+ // Session-level parquet page cache options
+ ParquetPageReadContext _ctx;
+
tparquet::PageHeader _cur_page_header;
+ bool _is_cache_payload_decompressed = true;
};
template <bool IN_COLLECTION, bool OFFSET_INDEX>
std::unique_ptr<PageReader<IN_COLLECTION, OFFSET_INDEX>> create_page_reader(
io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length,
- size_t total_rows, const tparquet::OffsetIndex* offset_index = nullptr) {
- return std::make_unique<PageReader<IN_COLLECTION, OFFSET_INDEX>>(reader, io_ctx, offset, length,
- total_rows, offset_index);
+ size_t total_rows, const tparquet::ColumnMetaData& metadata,
+ const tparquet::OffsetIndex* offset_index = nullptr,
+ const ParquetPageReadContext& ctx = ParquetPageReadContext()) {
+ return std::make_unique<PageReader<IN_COLLECTION, OFFSET_INDEX>>(
+ reader, io_ctx, offset, length, total_rows, metadata, offset_index, ctx);
}
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 76922608c2a..62dde51f009 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -183,6 +183,22 @@ void ParquetReader::_init_profile() {
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecompressTime", parquet_profile, 1);
_parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_read_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageReadCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheWriteCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_compressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheCompressedWriteCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_decompressed_write_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheDecompressedWriteCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheHitCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_missing_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheMissingCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_compressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheCompressedHitCount", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.page_cache_decompressed_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "PageCacheDecompressedHitCount", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.decode_header_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PageHeaderDecodeTime", parquet_profile, 1);
_parquet_profile.read_page_header_time =
@@ -789,12 +805,13 @@ Status ParquetReader::_next_row_group_reader() {
_profile, _file_reader, io_ranges, merged_read_slice_size)
: _file_reader;
}
- _current_group_reader.reset(new RowGroupReader(
- _io_ctx ? std::make_shared<io::TracingFileReader>(group_file_reader,
- _io_ctx->file_reader_stats)
- : group_file_reader,
- _read_table_columns, _current_row_group_index.row_group_id, row_group, _ctz, _io_ctx,
- position_delete_ctx, _lazy_read_ctx, _state, _column_ids, _filter_column_ids));
+ _current_group_reader.reset(
+ new RowGroupReader(_io_ctx ? std::make_shared<io::TracingFileReader>(
+ group_file_reader, _io_ctx->file_reader_stats)
+ : group_file_reader,
+ _read_table_columns, _current_row_group_index.row_group_id,
+ row_group, _ctz, _io_ctx, position_delete_ctx, _lazy_read_ctx,
+ _state, _column_ids, _filter_column_ids, _t_metadata->created_by));
_row_group_eof = false;
_current_group_reader->set_current_row_group_idx(_current_row_group_index);
@@ -1209,6 +1226,21 @@ void ParquetReader::_collect_profile() {
_column_statistics.page_index_read_calls);
COUNTER_UPDATE(_parquet_profile.decompress_time, _column_statistics.decompress_time);
COUNTER_UPDATE(_parquet_profile.decompress_cnt, _column_statistics.decompress_cnt);
+ COUNTER_UPDATE(_parquet_profile.page_read_counter, _column_statistics.page_read_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_write_counter,
+ _column_statistics.page_cache_write_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_compressed_write_counter,
+ _column_statistics.page_cache_compressed_write_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_decompressed_write_counter,
+ _column_statistics.page_cache_decompressed_write_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_hit_counter,
+ _column_statistics.page_cache_hit_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_missing_counter,
+ _column_statistics.page_cache_missing_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_compressed_hit_counter,
+ _column_statistics.page_cache_compressed_hit_counter);
+ COUNTER_UPDATE(_parquet_profile.page_cache_decompressed_hit_counter,
+ _column_statistics.page_cache_decompressed_hit_counter);
COUNTER_UPDATE(_parquet_profile.decode_header_time, _column_statistics.decode_header_time);
COUNTER_UPDATE(_parquet_profile.read_page_header_time,
_column_statistics.read_page_header_time);
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h
index 6652fe73092..a7a90846321 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -190,6 +190,14 @@ private:
RuntimeProfile::Counter* file_footer_hit_cache = nullptr;
RuntimeProfile::Counter* decompress_time = nullptr;
RuntimeProfile::Counter* decompress_cnt = nullptr;
+ RuntimeProfile::Counter* page_read_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_write_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_compressed_write_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_decompressed_write_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_hit_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_missing_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_compressed_hit_counter = nullptr;
+ RuntimeProfile::Counter* page_cache_decompressed_hit_counter = nullptr;
RuntimeProfile::Counter* decode_header_time = nullptr;
RuntimeProfile::Counter* read_page_header_time = nullptr;
RuntimeProfile::Counter* decode_value_time = nullptr;
diff --git a/be/test/io/fs/buffered_reader_test.cpp b/be/test/io/fs/buffered_reader_test.cpp
index 3874b06c68c..bc92d22b178 100644
--- a/be/test/io/fs/buffered_reader_test.cpp
+++ b/be/test/io/fs/buffered_reader_test.cpp
@@ -68,6 +68,8 @@ public:
bool closed() const override { return _reader->closed(); }
+ int64_t mtime() const override { return _reader->mtime(); }
+
private:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const io::IOContext* io_ctx) override {
@@ -96,6 +98,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return 0; }
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const io::IOContext* io_ctx) override {
@@ -130,6 +134,8 @@ public:
bool closed() const override { return _delegate->closed(); }
+ int64_t mtime() const override { return _delegate->mtime(); }
+
const io::PrefetchRange& last_read_range() const { return *_last_read_range; }
protected:
diff --git a/be/test/vec/exec/format/file_reader/file_meta_cache_test.cpp b/be/test/vec/exec/format/file_reader/file_meta_cache_test.cpp
index 3aef8db8459..5ea6335d2b9 100644
--- a/be/test/vec/exec/format/file_reader/file_meta_cache_test.cpp
+++ b/be/test/vec/exec/format/file_reader/file_meta_cache_test.cpp
@@ -38,6 +38,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return 0; }
+
Status close() override {
_closed = true;
return Status::OK();
diff --git a/be/test/vec/exec/format/parquet/parquet_page_cache_test.cpp b/be/test/vec/exec/format/parquet/parquet_page_cache_test.cpp
new file mode 100644
index 00000000000..21f2fa4e82b
--- /dev/null
+++ b/be/test/vec/exec/format/parquet/parquet_page_cache_test.cpp
@@ -0,0 +1,859 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <fmt/format.h>
+#include <gtest/gtest.h>
+
+#include "common/config.h"
+#include "io/fs/buffered_reader.h"
+#include "olap/page_cache.h"
+#include "runtime/exec_env.h"
+#include "runtime/memory/cache_manager.h"
+#include "util/block_compression.h"
+#include "util/faststring.h"
+#include "util/thrift_util.h"
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/parquet/vparquet_column_chunk_reader.h"
+#include "vec/exec/format/parquet/vparquet_page_reader.h"
+
+using namespace doris;
+using namespace doris::vectorized;
+
+class FakeBufferedReader : public io::BufferedStreamReader {
+public:
+ FakeBufferedReader(std::string path, std::vector<uint8_t> data)
+ : _path(std::move(path)), _data(std::move(data)) {}
+ Status read_bytes(const uint8_t** buf, uint64_t offset, const size_t bytes_to_read,
+ const doris::io::IOContext* io_ctx) override {
+ if (offset + bytes_to_read > _data.size()) return Status::IOError("Out of bounds");
+ *buf = _data.data() + offset;
+ return Status::OK();
+ }
+ Status read_bytes(Slice& slice, uint64_t offset, const doris::io::IOContext* io_ctx) override {
+ if (offset + slice.size > _data.size()) return Status::IOError("Out of bounds");
+ slice.data = reinterpret_cast<char*>(_data.data() + offset);
+ return Status::OK();
+ }
+ std::string path() override { return _path; }
+
+ int64_t mtime() const override { return 0; }
+
+private:
+ std::string _path;
+ std::vector<uint8_t> _data;
+};
+
+TEST(ParquetPageCacheTest, CacheHitReturnsDecompressedPayload) {
+ // setup storage page cache
+ // ExecEnv::GetInstance()->set_storage_page_cache(StoragePageCache::create_global_cache(1 << 20, 10, 0));
+
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+
+ // construct thrift PageHeader (uncompressed payload) and payload
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE;
+ header.__set_compressed_page_size(4);
+ header.__set_uncompressed_page_size(4);
+ header.__isset.data_page_header = true;
+ header.data_page_header.__set_num_values(1);
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> payload = {0x11, 0x22, 0x33, 0x44};
+ std::vector<uint8_t> cached_data;
+ cached_data.insert(cached_data.end(), header_bytes.begin(), header_bytes.end());
+ cached_data.insert(cached_data.end(), payload.begin(), payload.end());
+
+ std::string path = "test_parquet_cache_file";
+ int64_t header_offset = 128;
+ // make file_end_offset consistent with reader/page reader end offset used in test
+ int64_t file_end_offset = header_offset + static_cast<int64_t>(cached_data.size());
+
+ // insert into cache
+ int64_t mtime = 0;
+ StoragePageCache::CacheKey key(fmt::format("{}::{}", path, mtime),
+ static_cast<size_t>(file_end_offset), header_offset);
+ size_t total = cached_data.size();
+ auto* page = new DataPage(total, true, segment_v2::DATA_PAGE);
+ memcpy(page->data(), cached_data.data(), total);
+ page->reset_size(total);
+ PageCacheHandle handle;
+ StoragePageCache::instance()->insert(key, page, &handle, segment_v2::DATA_PAGE);
+
+ // create fake reader and a ColumnChunkReader to verify cache hit
+ // ensure the reader contains the same header+payload at the header offset so header parsing succeeds
+ std::vector<uint8_t> backing(256, 0);
+ memcpy(backing.data() + header_offset, cached_data.data(), total);
+ FakeBufferedReader reader(path, backing);
+ // prepare column chunk metadata so ColumnChunkReader uses same offsets
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(header_offset);
+ cc.meta_data.__set_total_compressed_size(total);
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+
+ FieldSchema field_schema;
+ field_schema.repetition_level = 0;
+ field_schema.definition_level = 0;
+
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &field_schema, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ // load_page_data should hit the cache and return decompressed payload
+ ASSERT_TRUE(ccr.load_page_data().ok());
+ Slice s = ccr.get_page_data();
+ ASSERT_EQ(s.size, payload.size());
+ ASSERT_EQ(0, memcmp(s.data, payload.data(), payload.size()));
+ // stats: ensure there was a page read and at least one hit recorded
+ auto& statistics = ccr.statistics();
+ EXPECT_EQ(statistics.page_read_counter, 1);
+ EXPECT_EQ(statistics.page_cache_hit_counter, 1);
+ EXPECT_EQ(statistics.page_cache_decompressed_hit_counter, 1);
+ // now safe to cleanup cache
+ // {
+ // StoragePageCache* _cache_ptr = StoragePageCache::instance();
+ // // Ensure any outstanding PageCacheHandle is released before destroying cache
+ // handle = PageCacheHandle();
+ // ExecEnv::GetInstance()->set_storage_page_cache(nullptr);
+ // delete _cache_ptr;
+ // // Clear CacheManager registration so tests do not leave global state
+ // // behind when unregister_cache is a no-op under BE_TEST.
+ // CacheManager::instance()->clear_for_test();
+ // }
+}
+
+TEST(ParquetPageCacheTest, DecompressedPageInsertedByColumnChunkReader) {
+ // ExecEnv::GetInstance()->set_storage_page_cache(StoragePageCache::create_global_cache(1 << 20, 10, 0));
+
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // ensure decompressed pages are cached via BE config
+ double old_thresh = config::parquet_page_cache_decompress_threshold;
+ bool old_enable_compressed = config::enable_parquet_cache_compressed_pages;
+ config::parquet_page_cache_decompress_threshold = 100.0;
+ config::enable_parquet_cache_compressed_pages = false;
+
+ // construct uncompressed header + payload in file buffer
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE;
+ header.__set_compressed_page_size(4);
+ header.__set_uncompressed_page_size(4);
+ header.__isset.data_page_header = true;
+ header.data_page_header.__set_num_values(1);
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> payload = {0x55, 0x66, 0x77, 0x88};
+ std::vector<uint8_t> file_data;
+ file_data.insert(file_data.end(), header_bytes.begin(), header_bytes.end());
+ file_data.insert(file_data.end(), payload.begin(), payload.end());
+
+ std::string path = "test_parquet_insert_file";
+ int64_t header_offset = 0;
+
+ FakeBufferedReader reader(path, file_data);
+
+ // prepare column chunk metadata
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(header_offset);
+ cc.meta_data.__set_total_compressed_size(file_data.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+
+ {
+ FieldSchema field_schema;
+ field_schema.repetition_level = 0;
+ field_schema.definition_level = 0;
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &field_schema, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ ASSERT_TRUE(ccr.load_page_data().ok());
+
+ // Now cache should have an entry; verify by creating a fresh ColumnChunkReader and hitting cache
+ ColumnChunkReader<false, false> ccr_check(&reader, &cc, &field_schema, nullptr, 0, nullptr,
+ ctx);
+ ASSERT_TRUE(ccr_check.init().ok());
+ // ASSERT_TRUE(ccr_check.next_page().ok());
+ ASSERT_TRUE(ccr_check.load_page_data().ok());
+ Slice s = ccr_check.get_page_data();
+ ASSERT_EQ(s.size, payload.size());
+ EXPECT_EQ(0, memcmp(s.data, payload.data(), payload.size()));
+ EXPECT_EQ(ccr_check.statistics().page_cache_hit_counter, 1);
+ }
+ // cleanup cache after readers go out of scope
+ // {
+ // StoragePageCache* _cache_ptr = StoragePageCache::instance();
+ // ExecEnv::GetInstance()->set_storage_page_cache(nullptr);
+ // delete _cache_ptr;
+ // CacheManager::instance()->clear_for_test();
+ // }
+ // restore config
+ config::parquet_page_cache_decompress_threshold = old_thresh;
+ config::enable_parquet_cache_compressed_pages = old_enable_compressed;
+}
+
+TEST(ParquetPageCacheTest, V2LevelsPreservedInCache) {
+ // ExecEnv::GetInstance()->set_storage_page_cache(StoragePageCache::create_global_cache(1 << 20, 10, 0));
+
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // ensure decompressed pages are cached via BE config
+ double old_thresh = config::parquet_page_cache_decompress_threshold;
+ bool old_enable_compressed = config::enable_parquet_cache_compressed_pages;
+ config::parquet_page_cache_decompress_threshold = 100.0;
+ config::enable_parquet_cache_compressed_pages = false;
+
+ // construct v2 header + levels + payload in file buffer (uncompressed)
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE_V2;
+ int rl = 2;
+ int dl = 1;
+ int payload_sz = 2;
+ header.__set_compressed_page_size(rl + dl + payload_sz);
+ header.__set_uncompressed_page_size(rl + dl + payload_sz);
+ header.__isset.data_page_header_v2 = true;
+ header.data_page_header_v2.__set_repetition_levels_byte_length(rl);
+ header.data_page_header_v2.__set_definition_levels_byte_length(dl);
+ header.data_page_header_v2.__set_is_compressed(false);
+ header.data_page_header_v2.__set_num_values(1);
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> level_bytes = {0x11, 0x22, 0x33};
+ std::vector<uint8_t> payload = {0xAA, 0xBB};
+ std::vector<uint8_t> file_data;
+ file_data.insert(file_data.end(), header_bytes.begin(), header_bytes.end());
+ file_data.insert(file_data.end(), level_bytes.begin(), level_bytes.end());
+ file_data.insert(file_data.end(), payload.begin(), payload.end());
+
+ std::string path = "test_v2_levels_file";
+ FakeBufferedReader reader(path, file_data);
+
+ // prepare column chunk metadata
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(0);
+ cc.meta_data.__set_total_compressed_size(file_data.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+
+ FieldSchema field_schema;
+ field_schema.repetition_level = 0;
+ field_schema.definition_level = 0;
+ {
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &field_schema, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ ASSERT_TRUE(ccr.load_page_data().ok());
+
+ // Now cache should have entry; verify by creating a ColumnChunkReader and hitting cache
+ ColumnChunkReader<false, false> ccr_check(&reader, &cc, &field_schema, nullptr, 0, nullptr,
+ ctx);
+ ASSERT_TRUE(ccr_check.init().ok());
+ // ASSERT_TRUE(ccr_check.next_page().ok());
+ ASSERT_TRUE(ccr_check.load_page_data().ok());
+ Slice s = ccr_check.get_page_data();
+ ASSERT_EQ(s.size, payload.size());
+ EXPECT_EQ(0, memcmp(s.data, payload.data(), payload.size()));
+ }
+
+ // Verify that a fresh ColumnChunkReader reusing cache gets level bytes preserved
+ FieldSchema field_schema2;
+ field_schema2.repetition_level = 2; // v2 levels present
+ field_schema2.definition_level = 1;
+ ColumnChunkReader<false, false> ccr2(&reader, &cc, &field_schema2, nullptr, 0, nullptr);
+ ASSERT_TRUE(ccr2.init().ok());
+ // ASSERT_TRUE(ccr2.next_page().ok());
+ ASSERT_TRUE(ccr2.load_page_data().ok());
+ // Level slices should equal the original level bytes
+ const Slice& rep = ccr2.v2_rep_levels();
+ const Slice& def = ccr2.v2_def_levels();
+ auto& statistics = ccr2.statistics();
+ EXPECT_GT(statistics.page_cache_hit_counter, 0);
+ // because threshold is set to cache decompressed, we should see decompressed hits
+ EXPECT_GT(statistics.page_cache_decompressed_hit_counter, 0);
+ ASSERT_EQ(def.size, dl);
+ EXPECT_EQ(0, memcmp(rep.data, level_bytes.data(), rl));
+ EXPECT_EQ(0, memcmp(def.data, level_bytes.data() + rl, dl));
+ // // cleanup cache after readers have been destroyed
+ // {
+ // StoragePageCache* _cache_ptr = StoragePageCache::instance();
+ // ExecEnv::GetInstance()->set_storage_page_cache(nullptr);
+ // delete _cache_ptr;
+ // CacheManager::instance()->clear_for_test();
+ // }
+ // restore config
+ config::parquet_page_cache_decompress_threshold = old_thresh;
+ config::enable_parquet_cache_compressed_pages = old_enable_compressed;
+}
+
+TEST(ParquetPageCacheTest, CompressedV1PageCachedAndHit) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // Note: parquet_page_cache_decompress_threshold and enable_parquet_cache_compressed_pages
+ // are now BE config variables, not context fields
+
+ // construct compressed v1 header + compressed payload in file buffer
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE;
+ header.__isset.data_page_header = true;
+ header.data_page_header.__set_num_values(1);
+
+ std::vector<uint8_t> payload = {0x01, 0x02, 0x03, 0x04};
+
+ // compress payload using a block codec
+ BlockCompressionCodec* codec = nullptr;
+ ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok());
+ faststring compressed_fast;
+ std::vector<Slice> inputs;
+ inputs.emplace_back(payload.data(), payload.size());
+ ASSERT_TRUE(codec->compress(inputs, payload.size(), &compressed_fast).ok());
+
+ header.__set_compressed_page_size(static_cast<int32_t>(compressed_fast.size()));
+ header.__set_uncompressed_page_size(static_cast<int32_t>(payload.size()));
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> file_data;
+ file_data.insert(file_data.end(), header_bytes.begin(), header_bytes.end());
+ file_data.insert(file_data.end(), compressed_fast.data(),
+ compressed_fast.data() + compressed_fast.size());
+
+ std::string path = "test_compressed_v1_file";
+ FakeBufferedReader reader(path, file_data);
+
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(0);
+ cc.meta_data.__set_total_compressed_size(file_data.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY);
+
+ FieldSchema field_schema;
+ field_schema.repetition_level = 0;
+ field_schema.definition_level = 0;
+
+ // Load page to trigger decompression + cache insert
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &field_schema, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ ASSERT_TRUE(ccr.load_page_data().ok());
+ EXPECT_EQ(ccr.statistics().page_cache_write_counter, 1);
+
+ // Now verify a fresh reader hits the cache and returns payload
+ ColumnChunkReader<false, false> ccr_check(&reader, &cc, &field_schema, nullptr, 0, nullptr,
+ ctx);
+ ASSERT_TRUE(ccr_check.init().ok());
+ // ASSERT_TRUE(ccr_check.next_page().ok());
+ ASSERT_TRUE(ccr_check.load_page_data().ok());
+ Slice s = ccr_check.get_page_data();
+ ASSERT_EQ(s.size, payload.size());
+ EXPECT_EQ(0, memcmp(s.data, payload.data(), payload.size()));
+ EXPECT_EQ(ccr_check.statistics().page_cache_hit_counter, 1);
+}
+
+TEST(ParquetPageCacheTest, CompressedV2LevelsPreservedInCache) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // Note: parquet_page_cache_decompress_threshold and enable_parquet_cache_compressed_pages
+ // are now BE config variables, not context fields
+
+ // construct v2 header + levels + compressed payload in file buffer
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE_V2;
+ int rl = 2;
+ int dl = 1;
+ //int payload_sz = 2;
+ header.__isset.data_page_header_v2 = true;
+ header.data_page_header_v2.__set_repetition_levels_byte_length(rl);
+ header.data_page_header_v2.__set_definition_levels_byte_length(dl);
+ header.data_page_header_v2.__set_is_compressed(true);
+ header.data_page_header_v2.__set_num_values(1);
+
+ std::vector<uint8_t> level_bytes = {0x11, 0x22, 0x33};
+ std::vector<uint8_t> payload = {0xAA, 0xBB};
+
+ // compress payload
+ BlockCompressionCodec* codec = nullptr;
+ ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok());
+ faststring compressed_fast;
+ std::vector<Slice> inputs;
+ inputs.emplace_back(payload.data(), payload.size());
+ ASSERT_TRUE(codec->compress(inputs, payload.size(), &compressed_fast).ok());
+
+ // compressed page: levels (uncompressed) followed by compressed payload
+ std::vector<uint8_t> compressed_page;
+ compressed_page.insert(compressed_page.end(), level_bytes.begin(), level_bytes.end());
+ compressed_page.insert(compressed_page.end(), compressed_fast.data(),
+ compressed_fast.data() + compressed_fast.size());
+
+ header.__set_compressed_page_size(static_cast<int32_t>(compressed_page.size()));
+ header.__set_uncompressed_page_size(static_cast<int32_t>(level_bytes.size() + payload.size()));
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> file_data;
+ file_data.insert(file_data.end(), header_bytes.begin(), header_bytes.end());
+ file_data.insert(file_data.end(), compressed_page.begin(), compressed_page.end());
+
+ std::string path = "test_compressed_v2_file";
+ FakeBufferedReader reader(path, file_data);
+
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(0);
+ cc.meta_data.__set_total_compressed_size(file_data.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY);
+
+ FieldSchema field_schema;
+ field_schema.repetition_level = 0;
+ field_schema.definition_level = 0;
+
+ // Load page to trigger decompression + cache insert
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &field_schema, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ ASSERT_TRUE(ccr.load_page_data().ok());
+ EXPECT_EQ(ccr.statistics().page_cache_write_counter, 1);
+
+ // Now verify a fresh reader hits the cache and v2 levels are preserved
+ FieldSchema field_schema2;
+ field_schema2.repetition_level = rl;
+ field_schema2.definition_level = dl;
+ ColumnChunkReader<false, false> ccr_check(&reader, &cc, &field_schema2, nullptr, 0, nullptr,
+ ctx);
+ ASSERT_TRUE(ccr_check.init().ok());
+ // ASSERT_TRUE(ccr_check.next_page().ok());
+ ASSERT_TRUE(ccr_check.load_page_data().ok());
+ Slice s = ccr_check.get_page_data();
+ ASSERT_EQ(s.size, payload.size());
+ EXPECT_EQ(0, memcmp(s.data, payload.data(), payload.size()));
+ const Slice& rep = ccr_check.v2_rep_levels();
+ const Slice& def = ccr_check.v2_def_levels();
+ ASSERT_EQ(rep.size, rl);
+ ASSERT_EQ(def.size, dl);
+ // cached v2 page is stored decompressed (threshold=100), make sure counter reflects it
+ EXPECT_GT(ccr_check.statistics().page_cache_decompressed_hit_counter, 0);
+ EXPECT_EQ(0, memcmp(rep.data, level_bytes.data(), rl));
+ EXPECT_EQ(0, memcmp(def.data, level_bytes.data() + rl, dl));
+}
+
+TEST(ParquetPageCacheTest, MultiPagesMixedV1V2CacheHit) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // Note: parquet_page_cache_decompress_threshold and enable_parquet_cache_compressed_pages
+ // are now BE config variables, not context fields
+
+ // Prepare a v1 uncompressed page and a v2 uncompressed page and insert both into cache
+ std::string path = "test_multi_pages_file";
+
+ // v1 page
+ tparquet::PageHeader hdr1;
+ hdr1.type = tparquet::PageType::DATA_PAGE;
+ hdr1.__set_compressed_page_size(4);
+ hdr1.__set_uncompressed_page_size(4);
+ hdr1.__isset.data_page_header = true;
+ hdr1.data_page_header.__set_num_values(1);
+ std::vector<uint8_t> header1_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&hdr1, &header1_bytes).ok());
+ std::vector<uint8_t> payload1 = {0x10, 0x20, 0x30, 0x40};
+ std::vector<uint8_t> cached1;
+ cached1.insert(cached1.end(), header1_bytes.begin(), header1_bytes.end());
+ cached1.insert(cached1.end(), payload1.begin(), payload1.end());
+
+ // v2 page
+ tparquet::PageHeader hdr2;
+ hdr2.type = tparquet::PageType::DATA_PAGE_V2;
+ int rl = 2;
+ int dl = 1;
+ int payload2_sz = 2;
+ hdr2.__set_compressed_page_size(rl + dl + payload2_sz);
+ hdr2.__set_uncompressed_page_size(rl + dl + payload2_sz);
+ hdr2.__isset.data_page_header_v2 = true;
+ hdr2.data_page_header_v2.__set_repetition_levels_byte_length(rl);
+ hdr2.data_page_header_v2.__set_definition_levels_byte_length(dl);
+ hdr2.data_page_header_v2.__set_is_compressed(false);
+ hdr2.data_page_header_v2.__set_num_values(1);
+ std::vector<uint8_t> header2_bytes;
+ ASSERT_TRUE(ts.serialize(&hdr2, &header2_bytes).ok());
+ std::vector<uint8_t> level_bytes = {0x11, 0x22, 0x33};
+ std::vector<uint8_t> payload2 = {0xAA, 0xBB};
+ std::vector<uint8_t> cached2;
+ cached2.insert(cached2.end(), header2_bytes.begin(), header2_bytes.end());
+ cached2.insert(cached2.end(), level_bytes.begin(), level_bytes.end());
+ cached2.insert(cached2.end(), payload2.begin(), payload2.end());
+
+ // Insert both pages into cache under different header offsets
+ size_t total1 = cached1.size();
+ auto* page1 = new DataPage(total1, true, segment_v2::DATA_PAGE);
+ memcpy(page1->data(), cached1.data(), total1);
+ page1->reset_size(total1);
+ PageCacheHandle h1;
+ size_t header1_start = 128;
+ int64_t mtime = 0;
+ StoragePageCache::CacheKey key1(fmt::format("{}::{}", path, mtime),
+ static_cast<size_t>(header1_start + total1), header1_start);
+ StoragePageCache::instance()->insert(key1, page1, &h1, segment_v2::DATA_PAGE);
+
+ size_t total2 = cached2.size();
+ auto* page2 = new DataPage(total2, true, segment_v2::DATA_PAGE);
+ memcpy(page2->data(), cached2.data(), total2);
+ page2->reset_size(total2);
+ PageCacheHandle h2;
+ size_t header2_start = 256;
+ StoragePageCache::CacheKey key2(fmt::format("{}::{}", path, mtime),
+ static_cast<size_t>(header2_start + total2), header2_start);
+ StoragePageCache::instance()->insert(key2, page2, &h2, segment_v2::DATA_PAGE);
+
+ // Now create readers that would lookup those cache keys
+ // Reader1 must expose header+page bytes at offset header1_start
+ std::vector<uint8_t> reader_backing1(3000, 0);
+ memcpy(reader_backing1.data() + header1_start, cached1.data(), total1);
+ FakeBufferedReader reader1(path, reader_backing1);
+ tparquet::ColumnChunk cc1;
+ cc1.meta_data.__set_data_page_offset(128);
+ cc1.meta_data.__set_total_compressed_size(total1);
+ cc1.meta_data.__set_num_values(1);
+ cc1.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ FieldSchema field_schema1;
+ field_schema1.repetition_level = 0;
+ field_schema1.definition_level = 0;
+ ColumnChunkReader<false, false> ccr1(&reader1, &cc1, &field_schema1, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr1.init().ok());
+ // ASSERT_TRUE(ccr1.next_page().ok());
+ ASSERT_TRUE(ccr1.load_page_data().ok());
+ Slice s1 = ccr1.get_page_data();
+ ASSERT_EQ(s1.size, payload1.size());
+ EXPECT_EQ(0, memcmp(s1.data, payload1.data(), payload1.size()));
+
+ std::vector<uint8_t> reader_backing2(3000, 0);
+ memcpy(reader_backing2.data() + header2_start, cached2.data(), total2);
+ FakeBufferedReader reader2(path, reader_backing2);
+ tparquet::ColumnChunk cc2;
+ cc2.meta_data.__set_data_page_offset(256);
+ cc2.meta_data.__set_total_compressed_size(total2);
+ cc2.meta_data.__set_num_values(1);
+ cc2.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ FieldSchema field_schema2;
+ field_schema2.repetition_level = rl;
+ field_schema2.definition_level = dl;
+ ColumnChunkReader<false, false> ccr2(&reader2, &cc2, &field_schema2, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr2.init().ok());
+ // ASSERT_TRUE(ccr2.next_page().ok());
+ ASSERT_TRUE(ccr2.load_page_data().ok());
+ Slice s2 = ccr2.get_page_data();
+ ASSERT_EQ(s2.size, payload2.size());
+ EXPECT_EQ(0, memcmp(s2.data, payload2.data(), payload2.size()));
+ const Slice& rep = ccr2.v2_rep_levels();
+ const Slice& def = ccr2.v2_def_levels();
+ ASSERT_EQ(rep.size, rl);
+ ASSERT_EQ(def.size, dl);
+ EXPECT_EQ(0, memcmp(rep.data, level_bytes.data(), rl));
+ EXPECT_EQ(0, memcmp(def.data, level_bytes.data() + rl, dl));
+}
+
+TEST(ParquetPageCacheTest, CacheMissThenHit) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // Note: parquet_page_cache_decompress_threshold and enable_parquet_cache_compressed_pages
+ // are now BE config variables, not context fields
+
+ // uncompressed v1 page
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE;
+ header.__set_compressed_page_size(4);
+ header.__set_uncompressed_page_size(4);
+ header.__isset.data_page_header = true;
+ header.data_page_header.__set_num_values(1);
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+ std::vector<uint8_t> payload = {0xDE, 0xAD, 0xBE, 0xEF};
+ std::vector<uint8_t> backing(256, 0);
+ std::vector<uint8_t> cached;
+ cached.insert(cached.end(), header_bytes.begin(), header_bytes.end());
+ cached.insert(cached.end(), payload.begin(), payload.end());
+ int64_t header_offset = 64;
+ memcpy(backing.data() + header_offset, cached.data(), cached.size());
+
+ std::string path = "test_miss_then_hit";
+ FakeBufferedReader reader(path, backing);
+
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(header_offset);
+ cc.meta_data.__set_total_compressed_size(cached.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+
+ FieldSchema fs;
+ fs.repetition_level = 0;
+ fs.definition_level = 0;
+
+ // First reader: should not hit cache, but should write cache
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &fs, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ ASSERT_TRUE(ccr.load_page_data().ok());
+ auto& statistics = ccr.statistics();
+ EXPECT_EQ(statistics.page_cache_hit_counter, 0);
+ EXPECT_EQ(statistics.page_cache_write_counter, 1);
+
+ // Second reader: should hit cache
+ ColumnChunkReader<false, false> ccr2(&reader, &cc, &fs, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr2.init().ok());
+ // ASSERT_TRUE(ccr2.next_page().ok());
+ ASSERT_TRUE(ccr2.load_page_data().ok());
+ auto& statistics2 = ccr2.statistics();
+ EXPECT_EQ(statistics2.page_cache_hit_counter, 1);
+ EXPECT_EQ(statistics2.page_cache_decompressed_hit_counter, 1);
+}
+
+TEST(ParquetPageCacheTest, DecompressThresholdCachesCompressed) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // Note: enable_parquet_cache_compressed_pages is now a BE config variable, not a context field
+
+ // prepare a compressible payload (lots of zeros)
+ std::vector<uint8_t> payload(1024, 0);
+
+ // compress payload using snappy
+ BlockCompressionCodec* codec = nullptr;
+ ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok());
+ faststring compressed_fast;
+ std::vector<Slice> inputs;
+ inputs.emplace_back(payload.data(), payload.size());
+ ASSERT_TRUE(codec->compress(inputs, payload.size(), &compressed_fast).ok());
+
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE;
+ header.__set_compressed_page_size(static_cast<int32_t>(compressed_fast.size()));
+ header.__set_uncompressed_page_size(static_cast<int32_t>(payload.size()));
+ header.__isset.data_page_header = true;
+ header.data_page_header.__set_num_values(1);
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> file_data;
+ file_data.insert(file_data.end(), header_bytes.begin(), header_bytes.end());
+ file_data.insert(file_data.end(), compressed_fast.data(),
+ compressed_fast.data() + compressed_fast.size());
+
+ std::string path = "test_threshold_file_compressed";
+ FakeBufferedReader reader(path, file_data);
+
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(0);
+ cc.meta_data.__set_total_compressed_size(file_data.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY);
+
+ FieldSchema fs;
+ fs.repetition_level = 0;
+ fs.definition_level = 0;
+
+ // Case: very small threshold -> cache the compressed payload (smaller footprint)
+ double old_thresh = config::parquet_page_cache_decompress_threshold;
+ bool old_enable_compressed = config::enable_parquet_cache_compressed_pages;
+ config::parquet_page_cache_decompress_threshold = 0.1;
+ config::enable_parquet_cache_compressed_pages = true;
+ ColumnChunkReader<false, false> ccr_small_thresh(&reader, &cc, &fs, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr_small_thresh.init().ok());
+ // ASSERT_TRUE(ccr_small_thresh.next_page().ok());
+ ASSERT_TRUE(ccr_small_thresh.load_page_data().ok());
+ EXPECT_EQ(ccr_small_thresh.statistics().page_cache_write_counter, 1);
+
+ // Inspect cache entry: payload stored should be compressed size
+ PageCacheHandle handle_small;
+ size_t file_end = header_bytes.size() + compressed_fast.size();
+ int64_t mtime = 0;
+ StoragePageCache::CacheKey key_small(fmt::format("{}::{}", path, mtime),
+ /*file_end_offset*/ file_end, /*header_start*/ 0);
+ bool found_small =
+ StoragePageCache::instance()->lookup(key_small, &handle_small, segment_v2::DATA_PAGE);
+ ASSERT_TRUE(found_small);
+ Slice cached_small = handle_small.data();
+ size_t header_size = header_bytes.size();
+ size_t payload_in_cache_size = cached_small.size - header_size; // no levels here
+ ASSERT_EQ(payload_in_cache_size, compressed_fast.size());
+
+ // restore config
+ config::parquet_page_cache_decompress_threshold = old_thresh;
+ config::enable_parquet_cache_compressed_pages = old_enable_compressed;
+}
+
+TEST(ParquetPageCacheTest, DecompressThresholdCachesDecompressed) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ // Note: enable_parquet_cache_compressed_pages is now a BE config variable, not a context field
+
+ // prepare a compressible payload (lots of zeros)
+ std::vector<uint8_t> payload(1024, 0);
+
+ // compress payload using snappy
+ BlockCompressionCodec* codec = nullptr;
+ ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok());
+ faststring compressed_fast;
+ std::vector<Slice> inputs;
+ inputs.emplace_back(payload.data(), payload.size());
+ ASSERT_TRUE(codec->compress(inputs, payload.size(), &compressed_fast).ok());
+
+ tparquet::PageHeader header;
+ header.type = tparquet::PageType::DATA_PAGE;
+ header.__set_compressed_page_size(static_cast<int32_t>(compressed_fast.size()));
+ header.__set_uncompressed_page_size(static_cast<int32_t>(payload.size()));
+ header.__isset.data_page_header = true;
+ header.data_page_header.__set_num_values(1);
+
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&header, &header_bytes).ok());
+
+ std::vector<uint8_t> file_data;
+ file_data.insert(file_data.end(), header_bytes.begin(), header_bytes.end());
+ file_data.insert(file_data.end(), compressed_fast.data(),
+ compressed_fast.data() + compressed_fast.size());
+
+ std::string path = "test_threshold_file_decompressed";
+ FakeBufferedReader reader(path, file_data);
+
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(0);
+ cc.meta_data.__set_total_compressed_size(file_data.size());
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY);
+
+ FieldSchema fs;
+ fs.repetition_level = 0;
+ fs.definition_level = 0;
+
+ // Case: very large threshold -> cache decompressed payload
+ double old_thresh = config::parquet_page_cache_decompress_threshold;
+ bool old_enable_compressed = config::enable_parquet_cache_compressed_pages;
+ config::parquet_page_cache_decompress_threshold = 100.0;
+ config::enable_parquet_cache_compressed_pages = false;
+ ColumnChunkReader<false, false> ccr_large_thresh(&reader, &cc, &fs, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr_large_thresh.init().ok());
+ // ASSERT_TRUE(ccr_large_thresh.next_page().ok());
+ ASSERT_TRUE(ccr_large_thresh.load_page_data().ok());
+ EXPECT_EQ(ccr_large_thresh.statistics().page_cache_write_counter, 1);
+
+ // Inspect cache entry for large threshold: payload stored should be uncompressed size
+ PageCacheHandle handle_large;
+ size_t file_end = header_bytes.size() + compressed_fast.size();
+ int64_t mtime = 0;
+ StoragePageCache::CacheKey key_large(fmt::format("{}::{}", path, mtime),
+ /*file_end_offset*/ file_end, /*header_start*/ 0);
+ bool found_large =
+ StoragePageCache::instance()->lookup(key_large, &handle_large, segment_v2::DATA_PAGE);
+ ASSERT_TRUE(found_large);
+ Slice cached_large = handle_large.data();
+ size_t payload_in_cache_size_large = cached_large.size - header_bytes.size();
+ ASSERT_EQ(payload_in_cache_size_large, payload.size());
+
+ // Verify cache hit for a new reader (should hit the decompressed entry we just created)
+ ColumnChunkReader<false, false> ccr_check(&reader, &cc, &fs, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr_check.init().ok());
+ // ASSERT_TRUE(ccr_check.next_page().ok());
+ ASSERT_TRUE(ccr_check.load_page_data().ok());
+ EXPECT_EQ(ccr_check.statistics().page_cache_hit_counter, 1);
+ // restore config
+ config::parquet_page_cache_decompress_threshold = old_thresh;
+ config::enable_parquet_cache_compressed_pages = old_enable_compressed;
+}
+
+TEST(ParquetPageCacheTest, MultipleReadersShareCachedEntry) {
+ ParquetPageReadContext ctx;
+ ctx.enable_parquet_file_page_cache = true;
+ double old_thresh = config::parquet_page_cache_decompress_threshold;
+ bool old_enable_compressed = config::enable_parquet_cache_compressed_pages;
+ config::parquet_page_cache_decompress_threshold = 100.0;
+ config::enable_parquet_cache_compressed_pages = false;
+
+ // Create a v2 cached page and then instantiate multiple readers that hit the cache
+ std::string path = "test_shared_handles";
+ tparquet::PageHeader hdr;
+ hdr.type = tparquet::PageType::DATA_PAGE_V2;
+ int rl = 2;
+ int dl = 1;
+ hdr.__isset.data_page_header_v2 = true;
+ hdr.data_page_header_v2.__set_repetition_levels_byte_length(rl);
+ hdr.data_page_header_v2.__set_definition_levels_byte_length(dl);
+ hdr.data_page_header_v2.__set_is_compressed(false);
+ hdr.data_page_header_v2.__set_num_values(1);
+ std::vector<uint8_t> header_bytes;
+ ThriftSerializer ts(/*compact*/ true, /*initial*/ 256);
+ ASSERT_TRUE(ts.serialize(&hdr, &header_bytes).ok());
+ std::vector<uint8_t> level_bytes = {0x11, 0x22, 0x33};
+ std::vector<uint8_t> payload = {0x0A, 0x0B};
+ std::vector<uint8_t> cached;
+ cached.insert(cached.end(), header_bytes.begin(), header_bytes.end());
+ cached.insert(cached.end(), level_bytes.begin(), level_bytes.end());
+ cached.insert(cached.end(), payload.begin(), payload.end());
+
+ size_t total = cached.size();
+ auto* page = new DataPage(total, true, segment_v2::DATA_PAGE);
+ memcpy(page->data(), cached.data(), total);
+ page->reset_size(total);
+ PageCacheHandle handle;
+ size_t header_start = 512;
+ int64_t mtime = 0;
+ StoragePageCache::CacheKey key(fmt::format("{}::{}", path, mtime),
+ static_cast<size_t>(header_start + total), header_start);
+ StoragePageCache::instance()->insert(key, page, &handle, segment_v2::DATA_PAGE);
+
+ // Create multiple readers that will hit cache
+ const int N = 4;
+ for (int i = 0; i < N; ++i) {
+ std::vector<uint8_t> reader_backing(5000, 0);
+ memcpy(reader_backing.data() + header_start, cached.data(), total);
+ FakeBufferedReader reader(path, reader_backing);
+ tparquet::ColumnChunk cc;
+ cc.meta_data.__set_data_page_offset(512);
+ cc.meta_data.__set_total_compressed_size(total);
+ cc.meta_data.__set_num_values(1);
+ cc.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ FieldSchema fs;
+ fs.repetition_level = rl;
+ fs.definition_level = dl;
+ ColumnChunkReader<false, false> ccr(&reader, &cc, &fs, nullptr, 0, nullptr, ctx);
+ ASSERT_TRUE(ccr.init().ok());
+ // ASSERT_TRUE(ccr.next_page().ok());
+ ASSERT_TRUE(ccr.load_page_data().ok());
+ Slice s = ccr.get_page_data();
+ ASSERT_EQ(s.size, payload.size());
+ EXPECT_EQ(0, memcmp(s.data, payload.data(), payload.size()));
+ const Slice& rep = ccr.v2_rep_levels();
+ const Slice& def = ccr.v2_def_levels();
+ ASSERT_EQ(rep.size, rl);
+ ASSERT_EQ(def.size, dl);
+ }
+ // restore config
+ config::parquet_page_cache_decompress_threshold = old_thresh;
+ config::enable_parquet_cache_compressed_pages = old_enable_compressed;
+}
diff --git a/be/test/vec/exec/orc/orc_file_reader_test.cpp b/be/test/vec/exec/orc/orc_file_reader_test.cpp
index 9e1003c397f..4c71129cdbb 100644
--- a/be/test/vec/exec/orc/orc_file_reader_test.cpp
+++ b/be/test/vec/exec/orc/orc_file_reader_test.cpp
@@ -41,6 +41,8 @@ public:
bool closed() const override { return _closed; }
+ int64_t mtime() const override { return 0; }
+
void set_data(const std::string& data) { _data = data; }
protected:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 361fd904ea8..b33deddb695 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -496,6 +496,9 @@ public class SessionVariable implements Serializable, Writable {
public static final String SHOW_USER_DEFAULT_ROLE = "show_user_default_role";
public static final String ENABLE_PAGE_CACHE = "enable_page_cache";
+ public static final String ENABLE_PARQUET_FILE_PAGE_CACHE = "enable_parquet_file_page_cache";
+ // public static final String PARQUET_PAGE_CACHE_DECOMPRESS_THRESHOLD = "parquet_page_cache_decompress_threshold";
+ // public static final String ENABLE_PARQUET_CACHE_COMPRESSED_PAGES = "enable_parquet_cache_compressed_pages";
public static final String MINIDUMP_PATH = "minidump_path";
@@ -2154,6 +2157,27 @@ public class SessionVariable implements Serializable, Writable {
needForward = true)
public boolean enablePageCache = true;
+ @VariableMgr.VarAttr(
+ name = ENABLE_PARQUET_FILE_PAGE_CACHE,
+ description = {"控制是否启用 Parquet file page cache。默认为 true。",
+ "Controls whether to use Parquet file page cache. The default is true."},
+ needForward = true)
+ public boolean enableParquetFilePageCache = true;
+
+ // @VariableMgr.VarAttr(
+ // name = PARQUET_PAGE_CACHE_DECOMPRESS_THRESHOLD,
+ // description = {"决定是否缓存解压后 page 的阈值,默认 1.5。",
+ // "Threshold ratio to decide caching decompressed parquet page, default 1.5."},
+ // needForward = true)
+ // public double parquetPageCacheDecompressThreshold = 1.5;
+
+ // @VariableMgr.VarAttr(
+ // name = ENABLE_PARQUET_CACHE_COMPRESSED_PAGES,
+ // description = {"控制是否缓存压缩的 Parquet 页面,默认为 false",
+ // "Controls whether to cache compressed parquet pages. Default false."},
+ // needForward = true)
+ // public boolean enableParquetCacheCompressedPages = false;
+
@VariableMgr.VarAttr(name = ENABLE_FOLD_NONDETERMINISTIC_FN)
public boolean enableFoldNondeterministicFn = false;
@@ -4876,6 +4900,8 @@ public class SessionVariable implements Serializable, Writable {
tResult.setEnablePageCache(enablePageCache);
+ tResult.setEnableParquetFilePageCache(enableParquetFilePageCache);
+
tResult.setFileCacheBasePath(fileCacheBasePath);
tResult.setEnableInvertedIndexQuery(enableInvertedIndexQuery);
@@ -4889,6 +4915,7 @@ public class SessionVariable implements Serializable, Writable {
tResult.setEnableParquetLazyMat(enableParquetLazyMat);
tResult.setEnableOrcLazyMat(enableOrcLazyMat);
tResult.setEnableParquetFilterByMinMax(enableParquetFilterByMinMax);
+ tResult.setEnableParquetFilePageCache(enableParquetFilePageCache);
tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax);
tResult.setCheckOrcInitSargsSuccess(checkOrcInitSargsSuccess);
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 88f063988c9..67c64195b34 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -418,6 +418,10 @@ struct TQueryOptions {
180: optional i32 max_file_scanners_concurrency = 0;
181: optional i32 min_file_scanners_concurrency = 0;
+ // Parquet page cache session options
+ // Whether to enable parquet file page cache on BE for this query
+ 184: optional bool enable_parquet_file_page_cache = true;
+
// For cloud, to control if the content would be written into file cache
// In write path, to control if the content would be written into file cache.
// In read path, read from file cache or remote storage when execute query.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/03: [fix](profile) sort out parquet reader profile (#58895)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 22931c2e9220c84f1ba42a500ad66fd56582bf15
Author: Mingyu Chen (Rayner) <mo...@163.com>
AuthorDate: Wed Dec 17 18:05:56 2025 +0800
[fix](profile) sort out parquet reader profile (#58895)
Problem Summary:
Refine some metrics in parquet reader profile.
1. Rename some `Statistics` class name to make it readable. (There are
too many `Statistics` struct with same name)
2. Add `read page header timer` in parquet reader profile
3. fix issue of invalid check logic for `MergeRangeFileReader` when
setting prefetch buffer size
4. fix issue that data cache profile is incorrect for external table
can.
---
be/src/io/fs/buffered_reader.cpp | 3 -
be/src/io/fs/buffered_reader.h | 18 ---
.../parquet/vparquet_column_chunk_reader.cpp | 14 +--
.../format/parquet/vparquet_column_chunk_reader.h | 18 +--
.../exec/format/parquet/vparquet_column_reader.h | 72 +++++-------
.../exec/format/parquet/vparquet_group_reader.cpp | 6 +-
.../exec/format/parquet/vparquet_group_reader.h | 2 +-
.../exec/format/parquet/vparquet_page_reader.cpp | 6 +-
.../vec/exec/format/parquet/vparquet_page_reader.h | 8 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 125 +++++++++++----------
be/src/vec/exec/format/parquet/vparquet_reader.h | 15 ++-
be/src/vec/exec/scan/file_scanner.cpp | 1 -
12 files changed, 129 insertions(+), 159 deletions(-)
diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
index cae08d28417..62774160555 100644
--- a/be/src/io/fs/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -819,12 +819,10 @@ Status BufferedFileStreamReader::read_bytes(const uint8_t** buf, uint64_t offset
int64_t buf_remaining = _buf_end_offset - _buf_start_offset;
int64_t to_read = std::min(_buf_size - buf_remaining, _file_end_offset - _buf_end_offset);
int64_t has_read = 0;
- SCOPED_RAW_TIMER(&_statistics.read_time);
while (has_read < to_read) {
size_t loop_read = 0;
Slice result(_buf.get() + buf_remaining + has_read, to_read - has_read);
RETURN_IF_ERROR(_file->read_at(_buf_end_offset + has_read, result, &loop_read, io_ctx));
- _statistics.read_calls++;
if (loop_read == 0) {
break;
}
@@ -833,7 +831,6 @@ Status BufferedFileStreamReader::read_bytes(const uint8_t** buf, uint64_t offset
if (has_read != to_read) {
return Status::Corruption("Try to read {} bytes, but received {} bytes", to_read, has_read);
}
- _statistics.read_bytes += to_read;
_buf_end_offset += to_read;
*buf = _buf.get();
return Status::OK();
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index 6bcf634aef3..5fe07176235 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -225,7 +225,6 @@ public:
int64_t merged_io = 0;
int64_t request_bytes = 0;
int64_t merged_bytes = 0;
- int64_t apply_bytes = 0;
};
struct RangeCachedData {
@@ -299,9 +298,6 @@ public:
_merged_read_slice_size = READ_SLICE_SIZE;
}
- for (const PrefetchRange& range : _random_access_ranges) {
- _statistics.apply_bytes += range.end_offset - range.start_offset;
- }
if (_profile != nullptr) {
const char* random_profile = "MergedSmallIO";
ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1);
@@ -315,8 +311,6 @@ public:
random_profile, 1);
_merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES,
random_profile, 1);
- _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES,
- random_profile, 1);
}
}
@@ -359,7 +353,6 @@ protected:
COUNTER_UPDATE(_merged_io, _statistics.merged_io);
COUNTER_UPDATE(_request_bytes, _statistics.request_bytes);
COUNTER_UPDATE(_merged_bytes, _statistics.merged_bytes);
- COUNTER_UPDATE(_apply_bytes, _statistics.apply_bytes);
if (_reader != nullptr) {
_reader->collect_profile_before_close();
}
@@ -373,7 +366,6 @@ private:
RuntimeProfile::Counter* _merged_io = nullptr;
RuntimeProfile::Counter* _request_bytes = nullptr;
RuntimeProfile::Counter* _merged_bytes = nullptr;
- RuntimeProfile::Counter* _apply_bytes = nullptr;
int _search_read_range(size_t start_offset, size_t end_offset);
void _clean_cached_data(RangeCachedData& cached_data);
@@ -619,12 +611,6 @@ private:
*/
class BufferedStreamReader {
public:
- struct Statistics {
- int64_t read_time = 0;
- int64_t read_calls = 0;
- int64_t read_bytes = 0;
- };
-
/**
* Return the address of underlying buffer that locates the start of data between [offset, offset + bytes_to_read)
* @param buf the buffer address to save the start address of data
@@ -637,13 +623,9 @@ public:
* Save the data address to slice.data, and the slice.size is the bytes to read.
*/
virtual Status read_bytes(Slice& slice, uint64_t offset, const IOContext* io_ctx) = 0;
- Statistics& statistics() { return _statistics; }
virtual ~BufferedStreamReader() = default;
// return the file path
virtual std::string path() = 0;
-
-protected:
- Statistics _statistics;
};
class BufferedFileStreamReader : public BufferedStreamReader, public ProfileCollector {
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
index 61a25cafa26..a474b76a518 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp
@@ -188,8 +188,8 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
// check decompressed buffer size
_reserve_decompress_buf(uncompressed_size);
_page_data = Slice(_decompress_buf.get(), uncompressed_size);
- SCOPED_RAW_TIMER(&_statistics.decompress_time);
- _statistics.decompress_cnt++;
+ SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time);
+ _chunk_statistics.decompress_cnt++;
RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data));
} else {
// Don't need decompress
@@ -204,7 +204,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
// Initialize repetition level and definition level. Skip when level = 0, which means required field.
if (_max_rep_level > 0) {
- SCOPED_RAW_TIMER(&_statistics.decode_level_time);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time);
if (header->__isset.data_page_header_v2) {
RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level,
_remaining_rep_nums));
@@ -215,7 +215,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::load_page_data() {
}
}
if (_max_def_level > 0) {
- SCOPED_RAW_TIMER(&_statistics.decode_level_time);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time);
if (header->__isset.data_page_header_v2) {
RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level,
_remaining_def_nums));
@@ -255,7 +255,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::_decode_dict_page() {
const tparquet::PageHeader* header = nullptr;
RETURN_IF_ERROR(_page_reader->get_page_header(header));
DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type);
- SCOPED_RAW_TIMER(&_statistics.decode_dict_time);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time);
// Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.
// Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.
@@ -314,7 +314,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::skip_values(size_t num_va
}
_remaining_num_values -= num_values;
if (skip_data) {
- SCOPED_RAW_TIMER(&_statistics.decode_value_time);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time);
return _page_decoder->skip_values(num_values);
} else {
return Status::OK();
@@ -328,7 +328,7 @@ Status ColumnChunkReader<IN_COLLECTION, OFFSET_INDEX>::decode_values(
if (select_vector.num_values() == 0) {
return Status::OK();
}
- SCOPED_RAW_TIMER(&_statistics.decode_value_time);
+ SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time);
if (UNLIKELY((doris_column->is_column_dictionary() || is_dict_filter) && !_has_dict)) {
return Status::IOError("Not dictionary coded");
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
index 992bc823430..93d0f92ecec 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h
@@ -60,6 +60,7 @@ struct ColumnChunkReaderStatistics {
int64_t decode_level_time = 0;
int64_t skip_page_header_num = 0;
int64_t parse_page_header_num = 0;
+ int64_t read_page_header_time = 0;
};
/**
@@ -158,12 +159,15 @@ public:
// Get page decoder
Decoder* get_page_decoder() { return _page_decoder; }
- ColumnChunkReaderStatistics& statistics() {
- _statistics.decode_header_time = _page_reader->statistics().decode_header_time;
- _statistics.skip_page_header_num = _page_reader->statistics().skip_page_header_num;
- _statistics.parse_page_header_num = _page_reader->statistics().parse_page_header_num;
- _statistics.read_page_header_time = _page_reader->statistics().read_page_header_time;
- return _statistics;
+ ColumnChunkReaderStatistics& chunk_statistics() {
+ _chunk_statistics.decode_header_time = _page_reader->page_statistics().decode_header_time;
+ _chunk_statistics.skip_page_header_num =
+ _page_reader->page_statistics().skip_page_header_num;
+ _chunk_statistics.parse_page_header_num =
+ _page_reader->page_statistics().parse_page_header_num;
+ _chunk_statistics.read_page_header_time =
+ _page_reader->page_statistics().read_page_header_time;
+ return _chunk_statistics;
}
Status read_dict_values_to_column(MutableColumnPtr& doris_column) {
@@ -251,7 +255,7 @@ private:
// Map: encoding -> Decoder
// Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding
std::unordered_map<int, std::unique_ptr<Decoder>> _decoders;
- ColumnChunkReaderStatistics _statistics;
+ ColumnChunkReaderStatistics _chunk_statistics;
};
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index 98df81fb540..4a49473a69f 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -53,12 +53,9 @@ using ColumnString = ColumnStr<UInt32>;
class ParquetColumnReader {
public:
- struct Statistics {
- Statistics()
- : read_time(0),
- read_calls(0),
- page_index_read_calls(0),
- read_bytes(0),
+ struct ColumnStatistics {
+ ColumnStatistics()
+ : page_index_read_calls(0),
decompress_time(0),
decompress_cnt(0),
decode_header_time(0),
@@ -70,12 +67,8 @@ public:
parse_page_header_num(0),
read_page_header_time(0) {}
- Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReaderStatistics& cs,
- int64_t null_map_time)
- : read_time(fs.read_time),
- read_calls(fs.read_calls),
- page_index_read_calls(0),
- read_bytes(fs.read_bytes),
+ ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time)
+ : page_index_read_calls(0),
decompress_time(cs.decompress_time),
decompress_cnt(cs.decompress_cnt),
decode_header_time(cs.decode_header_time),
@@ -87,10 +80,7 @@ public:
parse_page_header_num(cs.parse_page_header_num),
read_page_header_time(cs.read_page_header_time) {}
- int64_t read_time;
- int64_t read_calls;
int64_t page_index_read_calls;
- int64_t read_bytes;
int64_t decompress_time;
int64_t decompress_cnt;
int64_t decode_header_time;
@@ -102,21 +92,18 @@ public:
int64_t parse_page_header_num;
int64_t read_page_header_time;
- void merge(Statistics& statistics) {
- read_time += statistics.read_time;
- read_calls += statistics.read_calls;
- read_bytes += statistics.read_bytes;
- page_index_read_calls += statistics.page_index_read_calls;
- decompress_time += statistics.decompress_time;
- decompress_cnt += statistics.decompress_cnt;
- decode_header_time += statistics.decode_header_time;
- decode_value_time += statistics.decode_value_time;
- decode_dict_time += statistics.decode_dict_time;
- decode_level_time += statistics.decode_level_time;
- decode_null_map_time += statistics.decode_null_map_time;
- skip_page_header_num += statistics.skip_page_header_num;
- parse_page_header_num += statistics.parse_page_header_num;
- read_page_header_time += statistics.read_page_header_time;
+ void merge(ColumnStatistics& col_statistics) {
+ page_index_read_calls += col_statistics.page_index_read_calls;
+ decompress_time += col_statistics.decompress_time;
+ decompress_cnt += col_statistics.decompress_cnt;
+ decode_header_time += col_statistics.decode_header_time;
+ decode_value_time += col_statistics.decode_value_time;
+ decode_dict_time += col_statistics.decode_dict_time;
+ decode_level_time += col_statistics.decode_level_time;
+ decode_null_map_time += col_statistics.decode_null_map_time;
+ skip_page_header_num += col_statistics.skip_page_header_num;
+ parse_page_header_num += col_statistics.parse_page_header_num;
+ read_page_header_time += col_statistics.read_page_header_time;
}
};
@@ -148,7 +135,7 @@ public:
const std::set<uint64_t>& filter_column_ids = {});
virtual const std::vector<level_t>& get_rep_level() const = 0;
virtual const std::vector<level_t>& get_def_level() const = 0;
- virtual Statistics statistics() = 0;
+ virtual ColumnStatistics column_statistics() = 0;
virtual void close() = 0;
virtual void reset_filter_map_index() = 0;
@@ -191,9 +178,8 @@ public:
MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) override;
const std::vector<level_t>& get_rep_level() const override { return _rep_levels; }
const std::vector<level_t>& get_def_level() const override { return _def_levels; }
- Statistics statistics() override {
- return Statistics(_stream_reader->statistics(), _chunk_reader->statistics(),
- _decode_null_map_time);
+ ColumnStatistics column_statistics() override {
+ return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time);
}
void close() override {}
@@ -307,7 +293,7 @@ public:
const std::vector<level_t>& get_def_level() const override {
return _element_reader->get_def_level();
}
- Statistics statistics() override { return _element_reader->statistics(); }
+ ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); }
void close() override {}
void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); }
@@ -338,9 +324,9 @@ public:
return _key_reader->get_def_level();
}
- Statistics statistics() override {
- Statistics kst = _key_reader->statistics();
- Statistics vst = _value_reader->statistics();
+ ColumnStatistics column_statistics() override {
+ ColumnStatistics kst = _key_reader->column_statistics();
+ ColumnStatistics vst = _value_reader->column_statistics();
kst.merge(vst);
return kst;
}
@@ -395,12 +381,12 @@ public:
return _child_readers.begin()->second->get_def_level();
}
- Statistics statistics() override {
- Statistics st;
+ ColumnStatistics column_statistics() override {
+ ColumnStatistics st;
for (const auto& column_name : _read_column_names) {
auto reader = _child_readers.find(column_name);
if (reader != _child_readers.end()) {
- Statistics cst = reader->second->statistics();
+ ColumnStatistics cst = reader->second->column_statistics();
st.merge(cst);
}
}
@@ -493,8 +479,8 @@ public:
}
// Implement required pure virtual methods from base class
- Statistics statistics() override {
- return Statistics(); // Return empty statistics
+ ColumnStatistics column_statistics() override {
+ return ColumnStatistics(); // Return empty statistics
}
void close() override {
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index ad3a322e475..31161f2c6ed 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -1128,10 +1128,10 @@ void RowGroupReader::_convert_dict_cols_to_string_cols(Block* block) {
}
}
-ParquetColumnReader::Statistics RowGroupReader::statistics() {
- ParquetColumnReader::Statistics st;
+ParquetColumnReader::ColumnStatistics RowGroupReader::merged_column_statistics() {
+ ParquetColumnReader::ColumnStatistics st;
for (auto& reader : _column_readers) {
- auto ost = reader.second->statistics();
+ auto ost = reader.second->column_statistics();
st.merge(ost);
}
return st;
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
index f81d6607349..e9c4470af17 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
@@ -173,7 +173,7 @@ public:
int64_t predicate_filter_time() const { return _predicate_filter_time; }
int64_t dict_filter_rewrite_time() const { return _dict_filter_rewrite_time; }
- ParquetColumnReader::Statistics statistics();
+ ParquetColumnReader::ColumnStatistics merged_column_statistics();
void set_remaining_rows(int64_t rows) { _remaining_rows = rows; }
int64_t get_remaining_rows() { return _remaining_rows; }
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
index 19d4edf6d26..3b6d7fdcb9b 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp
@@ -88,11 +88,11 @@ Status PageReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
}
header_size = std::min(header_size, max_size);
{
- SCOPED_RAW_TIMER(&_statistics.read_page_header_time);
+ SCOPED_RAW_TIMER(&_page_statistics.read_page_header_time);
RETURN_IF_ERROR(_reader->read_bytes(&page_header_buf, _offset, header_size, _io_ctx));
}
real_header_size = cast_set<uint32_t>(header_size);
- SCOPED_RAW_TIMER(&_statistics.decode_header_time);
+ SCOPED_RAW_TIMER(&_page_statistics.decode_header_time);
auto st =
deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header);
if (st.ok()) {
@@ -115,7 +115,7 @@ Status PageReader<IN_COLLECTION, OFFSET_INDEX>::parse_page_header() {
}
}
- _statistics.parse_page_header_num++;
+ _page_statistics.parse_page_header_num++;
_offset += real_header_size;
_next_header_offset = _offset + _cur_page_header.compressed_page_size;
_state = HEADER_PARSED;
diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.h b/be/src/vec/exec/format/parquet/vparquet_page_reader.h
index 16dc49f030d..9246819d59c 100644
--- a/be/src/vec/exec/format/parquet/vparquet_page_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.h
@@ -53,7 +53,7 @@ namespace doris::vectorized {
template <bool IN_COLLECTION, bool OFFSET_INDEX>
class PageReader {
public:
- struct Statistics {
+ struct PageStatistics {
int64_t decode_header_time = 0;
int64_t skip_page_header_num = 0;
int64_t parse_page_header_num = 0;
@@ -83,7 +83,7 @@ public:
Status parse_page_header();
Status next_page() {
- _statistics.skip_page_header_num += _state == INITIALIZED;
+ _page_statistics.skip_page_header_num += _state == INITIALIZED;
if constexpr (OFFSET_INDEX) {
_page_index++;
_start_row = _offset_index->page_locations[_page_index].first_row_index;
@@ -133,7 +133,7 @@ public:
Status get_page_data(Slice& slice);
- Statistics& statistics() { return _statistics; }
+ PageStatistics& page_statistics() { return _page_statistics; }
bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; }
@@ -144,7 +144,7 @@ public:
private:
enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED };
PageReaderState _state = INITIALIZED;
- Statistics _statistics;
+ PageStatistics _page_statistics;
io::BufferedStreamReader* _reader = nullptr;
io::IOContext* _io_ctx = nullptr;
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 77d8212ab01..76922608c2a 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -140,9 +140,11 @@ void ParquetReader::_init_profile() {
ADD_TIMER_WITH_LEVEL(_profile, parquet_profile, 1);
_parquet_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
- _profile, "FilteredGroups", TUnit::UNIT, parquet_profile, 1);
+ _profile, "RowGroupsFiltered", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
- _profile, "ReadGroups", TUnit::UNIT, parquet_profile, 1);
+ _profile, "RowGroupsReadNum", TUnit::UNIT, parquet_profile, 1);
+ _parquet_profile.total_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
+ _profile, "RowGroupsTotalNum", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "FilteredRowsByGroup", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
@@ -153,16 +155,14 @@ void ParquetReader::_init_profile() {
_profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1);
_parquet_profile.raw_rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1);
- _parquet_profile.to_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(
- _profile, "ReadBytes", TUnit::BYTES, parquet_profile, 1);
_parquet_profile.column_read_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ColumnReadTime", parquet_profile, 1);
_parquet_profile.parse_meta_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ParseMetaTime", parquet_profile, 1);
_parquet_profile.parse_footer_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ParseFooterTime", parquet_profile, 1);
- _parquet_profile.open_file_time =
- ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileOpenTime", parquet_profile, 1);
+ _parquet_profile.file_reader_create_time =
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileReaderCreateTime", parquet_profile, 1);
_parquet_profile.open_file_num =
ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.page_index_read_calls =
@@ -184,7 +184,7 @@ void ParquetReader::_init_profile() {
_parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.decode_header_time =
- ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeHeaderTime", parquet_profile, 1);
+ ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PageHeaderDecodeTime", parquet_profile, 1);
_parquet_profile.read_page_header_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PageHeaderReadTime", parquet_profile, 1);
_parquet_profile.decode_value_time =
@@ -222,8 +222,8 @@ Status ParquetReader::_open_file() {
return Status::EndOfFile("stop");
}
if (_file_reader == nullptr) {
- SCOPED_RAW_TIMER(&_statistics.open_file_time);
- ++_statistics.open_file_num;
+ SCOPED_RAW_TIMER(&_reader_statistics.file_reader_create_time);
+ ++_reader_statistics.open_file_num;
_file_description.mtime =
_scan_range.__isset.modification_time ? _scan_range.modification_time : 0;
io::FileReaderOptions reader_options =
@@ -237,7 +237,7 @@ Status ParquetReader::_open_file() {
}
if (_file_metadata == nullptr) {
- SCOPED_RAW_TIMER(&_statistics.parse_footer_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.parse_footer_time);
if (_tracing_file_reader->size() <= sizeof(PARQUET_VERSION_NUMBER)) {
// Some system may generate parquet file with only 4 bytes: PAR1
// Should consider it as empty file.
@@ -251,9 +251,8 @@ Status ParquetReader::_open_file() {
&meta_size, _io_ctx));
_file_metadata = _file_metadata_ptr.get();
- _column_statistics.read_bytes += meta_size;
// parse magic number & parse meta data
- _statistics.file_footer_read_calls += 1;
+ _reader_statistics.file_footer_read_calls += 1;
} else {
const auto& file_meta_cache_key =
FileMetaCache::get_key(_tracing_file_reader, _file_description);
@@ -264,10 +263,9 @@ Status ParquetReader::_open_file() {
_meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(),
&_meta_cache_handle);
_file_metadata = _meta_cache_handle.data<FileMetaData>();
- _column_statistics.read_bytes += meta_size;
- _statistics.file_footer_read_calls += 1;
+ _reader_statistics.file_footer_read_calls += 1;
} else {
- _statistics.file_footer_hit_cache++;
+ _reader_statistics.file_footer_hit_cache++;
}
_file_metadata = _meta_cache_handle.data<FileMetaData>();
}
@@ -276,9 +274,6 @@ Status ParquetReader::_open_file() {
return Status::InternalError("failed to get file meta data: {}",
_file_description.path);
}
- _column_statistics.read_bytes += meta_size;
- // parse magic number & parse meta data
- _column_statistics.read_calls += 1;
}
return Status::OK();
}
@@ -337,7 +332,7 @@ Status ParquetReader::init_reader(
return Status::InternalError("failed to init parquet reader, please open reader first");
}
- SCOPED_RAW_TIMER(&_statistics.parse_meta_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.parse_meta_time);
_total_groups = _t_metadata->row_groups.size();
if (_total_groups == 0) {
return Status::EndOfFile("init reader failed, empty parquet file: " + _scan_range.path);
@@ -629,7 +624,7 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
return Status::OK();
}
- SCOPED_RAW_TIMER(&_statistics.column_read_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.column_read_time);
Status batch_st =
_current_group_reader->next_batch(block, _batch_size, read_rows, &_row_group_eof);
if (batch_st.is<ErrorCode::END_OF_FILE>()) {
@@ -646,11 +641,13 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
}
if (_row_group_eof) {
- auto column_st = _current_group_reader->statistics();
+ auto column_st = _current_group_reader->merged_column_statistics();
_column_statistics.merge(column_st);
- _statistics.lazy_read_filtered_rows += _current_group_reader->lazy_read_filtered_rows();
- _statistics.predicate_filter_time += _current_group_reader->predicate_filter_time();
- _statistics.dict_filter_rewrite_time += _current_group_reader->dict_filter_rewrite_time();
+ _reader_statistics.lazy_read_filtered_rows +=
+ _current_group_reader->lazy_read_filtered_rows();
+ _reader_statistics.predicate_filter_time += _current_group_reader->predicate_filter_time();
+ _reader_statistics.dict_filter_rewrite_time +=
+ _current_group_reader->dict_filter_rewrite_time();
if (_current_row_group_index.row_group_id + 1 == _total_groups) {
*eof = true;
} else {
@@ -742,23 +739,22 @@ Status ParquetReader::_next_row_group_reader() {
group_size += column_compressed_size(field);
}
- _statistics.read_rows += candidate_row_ranges.count();
+ _reader_statistics.read_rows += candidate_row_ranges.count();
if (_io_ctx) {
_io_ctx->file_reader_stats->read_rows += candidate_row_ranges.count();
}
if (candidate_row_ranges.count() != 0) {
// need read this row group.
- _statistics.read_row_groups++;
- _statistics.read_bytes += group_size;
-
- _statistics.filtered_page_rows += row_group.num_rows - candidate_row_ranges.count();
+ _reader_statistics.read_row_groups++;
+ _reader_statistics.filtered_page_rows +=
+ row_group.num_rows - candidate_row_ranges.count();
break;
} else {
// this row group be filtered.
- _statistics.filtered_row_groups++;
- _statistics.filtered_bytes += group_size;
- _statistics.filtered_group_rows += row_group.num_rows;
+ _reader_statistics.filtered_row_groups++;
+ _reader_statistics.filtered_bytes += group_size;
+ _reader_statistics.filtered_group_rows += row_group.num_rows;
}
}
@@ -929,11 +925,10 @@ Status ParquetReader::_process_page_index_filter(
Slice res(off_index_buff.data(), page_index._offset_index_size);
size_t bytes_read = 0;
{
- SCOPED_RAW_TIMER(&_statistics.read_page_index_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.read_page_index_time);
RETURN_IF_ERROR(_tracing_file_reader->read_at(page_index._offset_index_start, res,
&bytes_read, _io_ctx));
}
- _column_statistics.read_bytes += bytes_read;
_column_statistics.page_index_read_calls++;
_col_offsets.clear();
@@ -943,7 +938,7 @@ Status ParquetReader::_process_page_index_filter(
continue;
}
tparquet::OffsetIndex offset_index;
- SCOPED_RAW_TIMER(&_statistics.parse_page_index_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.parse_page_index_time);
RETURN_IF_ERROR(
page_index.parse_offset_index(chunk, off_index_buff.data(), &offset_index));
_col_offsets[parquet_col_id] = offset_index;
@@ -965,14 +960,13 @@ Status ParquetReader::_process_page_index_filter(
size_t bytes_read = 0;
Slice result(col_index_buff.data(), page_index._column_index_size);
{
- SCOPED_RAW_TIMER(&_statistics.read_page_index_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.read_page_index_time);
RETURN_IF_ERROR(_tracing_file_reader->read_at(page_index._column_index_start, result,
&bytes_read, _io_ctx));
}
- _column_statistics.read_bytes += bytes_read;
_column_statistics.page_index_read_calls++;
- SCOPED_RAW_TIMER(&_statistics.page_index_filter_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.page_index_filter_time);
// Construct a cacheable page index structure to avoid repeatedly reading the page index of the same column.
ParquetPredicate::CachedPageIndexStat cached_page_index;
@@ -1009,7 +1003,7 @@ Status ParquetReader::_process_page_index_filter(
tparquet::ColumnIndex column_index;
{
- SCOPED_RAW_TIMER(&_statistics.parse_page_index_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.parse_page_index_time);
RETURN_IF_ERROR(page_index.parse_column_index(column_chunk, col_index_buff.data(),
&column_index));
}
@@ -1072,7 +1066,7 @@ Status ParquetReader::_process_min_max_bloom_filter(
const RowGroupReader::RowGroupIndex& row_group_index, const tparquet::RowGroup& row_group,
const std::vector<std::unique_ptr<MutilColumnBlockPredicate>>& push_down_pred,
RowRanges* row_ranges) {
- SCOPED_RAW_TIMER(&_statistics.row_group_filter_time);
+ SCOPED_RAW_TIMER(&_reader_statistics.row_group_filter_time);
if (!_filter_groups) {
// No row group filtering is needed;
// for example, Iceberg reads position delete files.
@@ -1177,31 +1171,40 @@ void ParquetReader::_collect_profile() {
if (_current_group_reader != nullptr) {
_current_group_reader->collect_profile_before_close();
}
- COUNTER_UPDATE(_parquet_profile.filtered_row_groups, _statistics.filtered_row_groups);
- COUNTER_UPDATE(_parquet_profile.to_read_row_groups, _statistics.read_row_groups);
- COUNTER_UPDATE(_parquet_profile.filtered_group_rows, _statistics.filtered_group_rows);
- COUNTER_UPDATE(_parquet_profile.filtered_page_rows, _statistics.filtered_page_rows);
- COUNTER_UPDATE(_parquet_profile.lazy_read_filtered_rows, _statistics.lazy_read_filtered_rows);
- COUNTER_UPDATE(_parquet_profile.filtered_bytes, _statistics.filtered_bytes);
- COUNTER_UPDATE(_parquet_profile.raw_rows_read, _statistics.read_rows);
- COUNTER_UPDATE(_parquet_profile.to_read_bytes, _statistics.read_bytes);
- COUNTER_UPDATE(_parquet_profile.column_read_time, _statistics.column_read_time);
- COUNTER_UPDATE(_parquet_profile.parse_meta_time, _statistics.parse_meta_time);
- COUNTER_UPDATE(_parquet_profile.parse_footer_time, _statistics.parse_footer_time);
- COUNTER_UPDATE(_parquet_profile.open_file_time, _statistics.open_file_time);
- COUNTER_UPDATE(_parquet_profile.open_file_num, _statistics.open_file_num);
- COUNTER_UPDATE(_parquet_profile.page_index_filter_time, _statistics.page_index_filter_time);
- COUNTER_UPDATE(_parquet_profile.read_page_index_time, _statistics.read_page_index_time);
- COUNTER_UPDATE(_parquet_profile.parse_page_index_time, _statistics.parse_page_index_time);
- COUNTER_UPDATE(_parquet_profile.row_group_filter_time, _statistics.row_group_filter_time);
- COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, _statistics.file_footer_read_calls);
- COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _statistics.file_footer_hit_cache);
+ COUNTER_UPDATE(_parquet_profile.filtered_row_groups, _reader_statistics.filtered_row_groups);
+ COUNTER_UPDATE(_parquet_profile.to_read_row_groups, _reader_statistics.read_row_groups);
+ COUNTER_UPDATE(_parquet_profile.total_row_groups, _total_groups);
+ COUNTER_UPDATE(_parquet_profile.filtered_group_rows, _reader_statistics.filtered_group_rows);
+ COUNTER_UPDATE(_parquet_profile.filtered_page_rows, _reader_statistics.filtered_page_rows);
+ COUNTER_UPDATE(_parquet_profile.lazy_read_filtered_rows,
+ _reader_statistics.lazy_read_filtered_rows);
+ COUNTER_UPDATE(_parquet_profile.filtered_bytes, _reader_statistics.filtered_bytes);
+ COUNTER_UPDATE(_parquet_profile.raw_rows_read, _reader_statistics.read_rows);
+ COUNTER_UPDATE(_parquet_profile.column_read_time, _reader_statistics.column_read_time);
+ COUNTER_UPDATE(_parquet_profile.parse_meta_time, _reader_statistics.parse_meta_time);
+ COUNTER_UPDATE(_parquet_profile.parse_footer_time, _reader_statistics.parse_footer_time);
+ COUNTER_UPDATE(_parquet_profile.file_reader_create_time,
+ _reader_statistics.file_reader_create_time);
+ COUNTER_UPDATE(_parquet_profile.open_file_num, _reader_statistics.open_file_num);
+ COUNTER_UPDATE(_parquet_profile.page_index_filter_time,
+ _reader_statistics.page_index_filter_time);
+ COUNTER_UPDATE(_parquet_profile.read_page_index_time, _reader_statistics.read_page_index_time);
+ COUNTER_UPDATE(_parquet_profile.parse_page_index_time,
+ _reader_statistics.parse_page_index_time);
+ COUNTER_UPDATE(_parquet_profile.row_group_filter_time,
+ _reader_statistics.row_group_filter_time);
+ COUNTER_UPDATE(_parquet_profile.file_footer_read_calls,
+ _reader_statistics.file_footer_read_calls);
+ COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache,
+ _reader_statistics.file_footer_hit_cache);
COUNTER_UPDATE(_parquet_profile.skip_page_header_num, _column_statistics.skip_page_header_num);
COUNTER_UPDATE(_parquet_profile.parse_page_header_num,
_column_statistics.parse_page_header_num);
- COUNTER_UPDATE(_parquet_profile.predicate_filter_time, _statistics.predicate_filter_time);
- COUNTER_UPDATE(_parquet_profile.dict_filter_rewrite_time, _statistics.dict_filter_rewrite_time);
+ COUNTER_UPDATE(_parquet_profile.predicate_filter_time,
+ _reader_statistics.predicate_filter_time);
+ COUNTER_UPDATE(_parquet_profile.dict_filter_rewrite_time,
+ _reader_statistics.dict_filter_rewrite_time);
COUNTER_UPDATE(_parquet_profile.page_index_read_calls,
_column_statistics.page_index_read_calls);
COUNTER_UPDATE(_parquet_profile.decompress_time, _column_statistics.decompress_time);
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h
index 3d08632102c..6652fe73092 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -74,7 +74,7 @@ class ParquetReader : public GenericReader, public ExprPushDownHelper {
ENABLE_FACTORY_CREATOR(ParquetReader);
public:
- struct Statistics {
+ struct ReaderStatistics {
int32_t filtered_row_groups = 0;
int32_t read_row_groups = 0;
int64_t filtered_group_rows = 0;
@@ -82,13 +82,12 @@ public:
int64_t lazy_read_filtered_rows = 0;
int64_t read_rows = 0;
int64_t filtered_bytes = 0;
- int64_t read_bytes = 0;
int64_t column_read_time = 0;
int64_t parse_meta_time = 0;
int64_t parse_footer_time = 0;
int64_t file_footer_read_calls = 0;
int64_t file_footer_hit_cache = 0;
- int64_t open_file_time = 0;
+ int64_t file_reader_create_time = 0;
int64_t open_file_num = 0;
int64_t row_group_filter_time = 0;
int64_t page_index_filter_time = 0;
@@ -141,7 +140,7 @@ public:
Status get_parsed_schema(std::vector<std::string>* col_names,
std::vector<DataTypePtr>* col_types) override;
- Statistics& statistics() { return _statistics; }
+ ReaderStatistics& reader_statistics() { return _reader_statistics; }
const tparquet::FileMetaData* get_meta_data() const { return _t_metadata; }
@@ -171,16 +170,16 @@ private:
struct ParquetProfile {
RuntimeProfile::Counter* filtered_row_groups = nullptr;
RuntimeProfile::Counter* to_read_row_groups = nullptr;
+ RuntimeProfile::Counter* total_row_groups = nullptr;
RuntimeProfile::Counter* filtered_group_rows = nullptr;
RuntimeProfile::Counter* filtered_page_rows = nullptr;
RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr;
RuntimeProfile::Counter* filtered_bytes = nullptr;
RuntimeProfile::Counter* raw_rows_read = nullptr;
- RuntimeProfile::Counter* to_read_bytes = nullptr;
RuntimeProfile::Counter* column_read_time = nullptr;
RuntimeProfile::Counter* parse_meta_time = nullptr;
RuntimeProfile::Counter* parse_footer_time = nullptr;
- RuntimeProfile::Counter* open_file_time = nullptr;
+ RuntimeProfile::Counter* file_reader_create_time = nullptr;
RuntimeProfile::Counter* open_file_num = nullptr;
RuntimeProfile::Counter* row_group_filter_time = nullptr;
RuntimeProfile::Counter* page_index_read_calls = nullptr;
@@ -318,8 +317,8 @@ private:
// _table_column_names = _missing_cols + _read_table_columns
const std::vector<std::string>* _table_column_names = nullptr;
- Statistics _statistics;
- ParquetColumnReader::Statistics _column_statistics;
+ ReaderStatistics _reader_statistics;
+ ParquetColumnReader::ColumnStatistics _column_statistics;
ParquetProfile _parquet_profile;
bool _closed = false;
io::IOContext* _io_ctx = nullptr;
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
index a2e74b3d547..3cc4ed9d7e5 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -1757,7 +1757,6 @@ void FileScanner::update_realtime_counters() {
_file_cache_statistics->bytes_read_from_local - _last_bytes_read_from_local;
int64_t delta_bytes_read_from_remote =
_file_cache_statistics->bytes_read_from_remote - _last_bytes_read_from_remote;
-
if (_file_cache_statistics->bytes_read_from_local == 0 &&
_file_cache_statistics->bytes_read_from_remote == 0) {
_state->get_query_ctx()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org