You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/11/01 14:53:04 UTC
[incubator-doris] branch master updated: Segment v2 stream load
core dump(#2037) (#2075)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e1a8f9d Segment v2 stream load core dump(#2037) (#2075)
e1a8f9d is described below
commit e1a8f9d30fce423e5d338e6dd17581b25911a994
Author: wangbo <50...@qq.com>
AuthorDate: Fri Nov 1 22:52:58 2019 +0800
Segment v2 stream load core dump(#2037) (#2075)
[STORAGE]
1 fix mem fix mem leak when calling string builder.get_dictionary_page;
2 fix delete invalid mem addr in bitshuffleBuilder when no array grow happends
when bitshuffleBuilder didn't grow array, the data page which not use new to allocate will be
returned to ColumnWriter.
When ColumnWriter destructs, the data page will be deleted,this causes core dump
---
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 13 ++--
be/src/olap/rowset/segment_v2/binary_dict_page.h | 16 ++---
be/src/olap/rowset/segment_v2/binary_plain_page.h | 19 ++---
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 22 +++---
be/src/olap/rowset/segment_v2/column_writer.cpp | 13 ++--
be/src/olap/rowset/segment_v2/column_writer.h | 7 +-
be/src/olap/rowset/segment_v2/column_zone_map.h | 2 +-
.../rowset/segment_v2/frame_of_reference_page.h | 18 ++---
be/src/olap/rowset/segment_v2/page_builder.h | 20 ++----
be/src/olap/rowset/segment_v2/plain_page.h | 18 ++---
be/src/olap/rowset/segment_v2/rle_page.h | 18 ++---
be/src/util/owned_slice.h | 80 ++++++++++++++++++++++
.../rowset/segment_v2/binary_dict_page_test.cpp | 34 +++++----
.../rowset/segment_v2/binary_plain_page_test.cpp | 6 +-
.../rowset/segment_v2/bitshuffle_page_test.cpp | 6 +-
.../rowset/segment_v2/column_zone_map_test.cpp | 8 +--
.../segment_v2/frame_of_reference_page_test.cpp | 14 ++--
be/test/olap/rowset/segment_v2/plain_page_test.cpp | 4 +-
be/test/olap/rowset/segment_v2/rle_page_test.cpp | 14 ++--
19 files changed, 187 insertions(+), 145 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index 61c9e36..5e2c5a0 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -99,13 +99,15 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals, size_t* count) {
}
}
-Slice BinaryDictPageBuilder::finish() {
+OwnedSlice BinaryDictPageBuilder::finish() {
_finished = true;
- Slice data_slice = _data_page_builder->finish();
- _buffer.append(data_slice.data, data_slice.size);
+ OwnedSlice data_slice(_data_page_builder->finish());
+ _buffer.append(data_slice.slice().data, data_slice.slice().size);
encode_fixed32_le(&_buffer[0], _encoding_type);
- return Slice(_buffer);
+
+ size_t buf_size = _buffer.size();
+ return OwnedSlice(_buffer.release(), buf_size);
}
void BinaryDictPageBuilder::reset() {
@@ -131,7 +133,7 @@ uint64_t BinaryDictPageBuilder::size() const {
return _pool.total_allocated_bytes() + _data_page_builder->size();
}
-Status BinaryDictPageBuilder::get_dictionary_page(Slice* dictionary_page) {
+Status BinaryDictPageBuilder::get_dictionary_page(OwnedSlice* dictionary_page) {
_dictionary.clear();
_dict_builder->reset();
size_t add_count = 1;
@@ -141,7 +143,6 @@ Status BinaryDictPageBuilder::get_dictionary_page(Slice* dictionary_page) {
RETURN_IF_ERROR(_dict_builder->add(reinterpret_cast<const uint8_t*>(&dict_item), &add_count));
}
*dictionary_page = _dict_builder->finish();
- _dict_builder->release();
_dict_items.clear();
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.h b/be/src/olap/rowset/segment_v2/binary_dict_page.h
index 6a2ef32..de24a93 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.h
@@ -58,7 +58,10 @@ public:
Status add(const uint8_t* vals, size_t* count) override;
- Slice finish() override;
+ // this api will release the memory ownership of encoded data
+ // Note:
+ // reset() should be called after this function before reuse the builder
+ OwnedSlice finish() override;
void reset() override;
@@ -66,16 +69,7 @@ public:
uint64_t size() const override;
- Status get_dictionary_page(Slice* dictionary_page) override;
-
- // this api will release the memory ownership of encoded data
- // Note:
- // release() should be called after finish
- // reset() should be called after this function before reuse the builder
- void release() override {
- uint8_t* ret = _buffer.release();
- (void)ret;
- }
+ Status get_dictionary_page(OwnedSlice* dictionary_page) override;
private:
PageBuilderOptions _options;
diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h
index b442eb8..de765d5 100644
--- a/be/src/olap/rowset/segment_v2/binary_plain_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h
@@ -30,6 +30,7 @@
#include "util/coding.h"
#include "util/faststring.h"
+#include "util/owned_slice.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/page_builder.h"
#include "olap/rowset/segment_v2/page_decoder.h"
@@ -78,7 +79,10 @@ public:
return Status::OK();
}
- Slice finish() override {
+ // this api will release the memory ownership of encoded data
+ // Note:
+ // reset() should be called after this function before reuse the builder
+ OwnedSlice finish() override {
_finished = true;
// Set up trailer
@@ -87,7 +91,8 @@ public:
}
put_fixed32_le(&_buffer, _offsets.size());
- return Slice(_buffer);
+ size_t buf_size = _buffer.size();
+ return OwnedSlice(_buffer.release(), buf_size);
}
void reset() override {
@@ -107,16 +112,6 @@ public:
return _size_estimate;
}
- // this api will release the memory ownership of encoded data
- // Note:
- // release() should be called after finish
- // reset() should be called after this function before reuse the builder
- void release() override {
- uint8_t* ret = _buffer.release();
- _buffer.reserve(_options.data_page_size);
- (void) ret;
- }
-
void update_prepared_size(size_t added_size) {
_prepared_size += added_size;
_prepared_size += sizeof(uint32_t);
diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
index 1573d68..6288d7d 100644
--- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h
+++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
@@ -26,6 +26,7 @@
#include "util/coding.h"
#include "util/faststring.h"
+#include "util/owned_slice.h"
#include "gutil/port.h"
#include "olap/olap_common.h"
#include "olap/types.h"
@@ -105,7 +106,10 @@ public:
return Status::OK();
}
- Slice finish() override {
+ // this api will release the memory ownership of encoded data
+ // Note:
+ // reset() should be called after this function before reuse the builder
+ OwnedSlice finish() override {
return _finish(SIZE_OF_TYPE);
}
@@ -130,17 +134,8 @@ public:
return _buffer.size();
}
- // this api will release the memory ownership of encoded data
- // Note:
- // release() should be called after finish
- // reset() should be called after this function before reuse the builder
- void release() override {
- uint8_t* ret = _buffer.release();
- (void)ret;
- }
-
private:
- Slice _finish(int final_size_of_type) {
+ OwnedSlice _finish(int final_size_of_type) {
_data.resize(final_size_of_type * _count);
// Do padding so that the input num of element is multiple of 8.
@@ -163,13 +158,14 @@ private:
warn_with_bitshuffle_error(bytes);
// It does not matter what will be returned here,
// since we have logged fatal in warn_with_bitshuffle_error().
- return Slice();
+ return OwnedSlice(Slice((const uint8_t*)nullptr, 0));
}
encode_fixed32_le(&_buffer[4], BITSHUFFLE_PAGE_HEADER_SIZE + bytes);
encode_fixed32_le(&_buffer[8], num_elems_after_padding);
encode_fixed32_le(&_buffer[12], final_size_of_type);
_finished = true;
- return Slice(_buffer.data(), BITSHUFFLE_PAGE_HEADER_SIZE + bytes);
+
+ return OwnedSlice(_buffer.release(), BITSHUFFLE_PAGE_HEADER_SIZE + bytes);
}
typedef typename TypeTraits<Type>::CppType CppType;
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index f14911e..0f8b585 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -186,7 +186,7 @@ uint64_t ColumnWriter::estimate_buffer_size() {
uint64_t size = 0;
Page* page = _pages.head;
while (page != nullptr) {
- size += page->data.get_size();
+ size += page->data.slice().size;
if (_is_nullable) {
size += page->null_bitmap.get_size();
}
@@ -215,10 +215,10 @@ Status ColumnWriter::write_data() {
}
// write column dict
if (_encoding_info->encoding() == DICT_ENCODING) {
- Slice dict_page;
+ OwnedSlice dict_page;
_page_builder->get_dictionary_page(&dict_page);
std::vector<Slice> origin_data;
- origin_data.push_back(dict_page);
+ origin_data.push_back(dict_page.slice());
RETURN_IF_ERROR(_write_physical_page(&origin_data, &_dict_page_pp));
}
return Status::OK();
@@ -232,8 +232,8 @@ Status ColumnWriter::write_ordinal_index() {
Status ColumnWriter::write_zone_map() {
if (_opts.need_zone_map) {
- Slice data = _column_zone_map_builder->finish();
- std::vector<Slice> slices{data};
+ OwnedSlice data(_column_zone_map_builder->finish());
+ std::vector<Slice> slices{data.slice()};
return _write_physical_page(&slices, &_zone_map_pp);
}
return Status::OK();
@@ -270,7 +270,7 @@ Status ColumnWriter::_write_data_page(Page* page) {
if (_is_nullable) {
origin_data.push_back(page->null_bitmap);
}
- origin_data.push_back(page->data);
+ origin_data.push_back(page->data.slice());
// TODO(zc): upadte page's statistics
PagePointer pp;
@@ -329,7 +329,6 @@ Status ColumnWriter::_finish_current_page() {
page->first_rowid = _last_first_rowid;
page->num_rows = _next_rowid - _last_first_rowid;
page->data = _page_builder->finish();
- _page_builder->release();
_page_builder->reset();
if (_is_nullable) {
page->null_bitmap = _null_bitmap_builder->finish();
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index 6626b4d..8783b34 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -22,7 +22,7 @@
#include "common/status.h" // for Status
#include "gen_cpp/segment_v2.pb.h" // for EncodingTypePB
#include "util/bitmap.h" // for BitmapChange
-#include "util/slice.h" // for slice
+#include "util/owned_slice.h" // for owned slice
#include "olap/rowset/segment_v2/common.h" // for rowid_t
#include "olap/rowset/segment_v2/page_pointer.h" // for PagePointer
#include "olap/rowset/segment_v2/column_zone_map.h" // for ColumnZoneMapBuilder
@@ -102,14 +102,13 @@ private:
struct Page {
int32_t first_rowid;
int32_t num_rows;
- Slice null_bitmap = Slice((uint8_t*)nullptr, 0);
- Slice data = Slice((uint8_t*)nullptr, 0);
+ Slice null_bitmap = Slice((const uint8_t*)nullptr, 0);
+ OwnedSlice data = OwnedSlice((uint8_t*)nullptr, 0);
Page* next = nullptr;
~Page() {
delete[] null_bitmap.data;
- delete[] data.data;
}
};
diff --git a/be/src/olap/rowset/segment_v2/column_zone_map.h b/be/src/olap/rowset/segment_v2/column_zone_map.h
index f4aaa45..6989725 100644
--- a/be/src/olap/rowset/segment_v2/column_zone_map.h
+++ b/be/src/olap/rowset/segment_v2/column_zone_map.h
@@ -64,7 +64,7 @@ public:
return _page_builder->size();
}
- Slice finish() {
+ OwnedSlice finish() {
return _page_builder->finish();
}
diff --git a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
index 6b33ccf..ad334e2 100644
--- a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
+++ b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
@@ -49,10 +49,15 @@ public:
return Status::OK();
}
- Slice finish() override {
+ // this api will release the memory ownership of encoded data
+ // Note:
+ // reset() should be called after this function before reuse the builder
+ OwnedSlice finish() override {
_finished = true;
_encoder->flush();
- return Slice(_buf.data(), _buf.size());
+
+ size_t buf_size = _buf.size();
+ return OwnedSlice(_buf.release(), buf_size);
}
void reset() override {
@@ -69,15 +74,6 @@ public:
return _buf.size();
}
- // this api will release the memory ownership of encoded data
- // Note:
- // release() should be called after finish
- // reset() should be called after this function before reuse the builder
- void release() override {
- uint8_t* ret = _buf.release();
- (void)ret;
- }
-
private:
typedef typename TypeTraits<Type>::CppType CppType;
PageBuilderOptions _options;
diff --git a/be/src/olap/rowset/segment_v2/page_builder.h b/be/src/olap/rowset/segment_v2/page_builder.h
index c2cc0eb..3248f34 100644
--- a/be/src/olap/rowset/segment_v2/page_builder.h
+++ b/be/src/olap/rowset/segment_v2/page_builder.h
@@ -21,7 +21,7 @@
#include <vector>
#include "gutil/macros.h"
-#include "util/slice.h"
+#include "util/owned_slice.h"
#include "common/status.h"
#include "olap/rowset/segment_v2/common.h"
@@ -52,16 +52,16 @@ public:
// vals size should be decided according to the page build type
virtual doris::Status add(const uint8_t* vals, size_t* count) = 0;
+ // This api is for release the resource owned by builder
+ // It means it will transfer the ownership of some resource to other.
+ // This api should be followed by reset() before reuse the builder
+ virtual OwnedSlice finish() = 0;
+
// Get the dictionary page for dictionary encoding mode column.
- virtual Status get_dictionary_page(Slice* dictionary_page) {
+ virtual Status get_dictionary_page(OwnedSlice* dictionary_page) {
return Status::NotSupported("get_dictionary_page not implemented");
}
- // Return a Slice which represents the encoded data of current page.
- //
- // This Slice points to internal data of this builder.
- virtual Slice finish() = 0;
-
// Reset the internal state of the page builder.
//
// Any data previously returned by finish may be invalidated by this call.
@@ -73,12 +73,6 @@ public:
// Return the total bytes of pageBuilder that have been added to the page.
virtual uint64_t size() const = 0;
- // This api is for release the resource owned by builder
- // It means it will transfer the ownership of some resource to other.
- // This api is always called after finish
- // and should be followed by reset() before reuse the builder
- virtual void release() = 0;
-
private:
DISALLOW_COPY_AND_ASSIGN(PageBuilder);
};
diff --git a/be/src/olap/rowset/segment_v2/plain_page.h b/be/src/olap/rowset/segment_v2/plain_page.h
index d182d17..459b0ed 100644
--- a/be/src/olap/rowset/segment_v2/plain_page.h
+++ b/be/src/olap/rowset/segment_v2/plain_page.h
@@ -57,12 +57,16 @@ public:
return Status::OK();
}
- Slice finish() override {
+ // this api will release the memory ownership of encoded data
+ // Note:
+ // reset() should be called after this function before reuse the builder
+ OwnedSlice finish() override {
encode_fixed32_le((uint8_t *) &_buffer[0], _count);
- return Slice(_buffer.data(), PLAIN_PAGE_HEADER_SIZE + _count * SIZE_OF_TYPE);
+ return OwnedSlice(_buffer.release(), PLAIN_PAGE_HEADER_SIZE + _count * SIZE_OF_TYPE);
}
void reset() override {
+ _buffer.reserve(_options.data_page_size + 1024);
_count = 0;
_buffer.clear();
_buffer.resize(PLAIN_PAGE_HEADER_SIZE);
@@ -76,16 +80,6 @@ public:
return _buffer.size();
}
- // this api will release the memory ownership of encoded data
- // Note:
- // release() should be called after finish
- // reset() should be called after this function before reuse the builder
- void release() override {
- uint8_t *ret = _buffer.release();
- _buffer.reserve(_options.data_page_size + 1024);
- (void) ret;
- }
-
private:
faststring _buffer;
PageBuilderOptions _options;
diff --git a/be/src/olap/rowset/segment_v2/rle_page.h b/be/src/olap/rowset/segment_v2/rle_page.h
index 46fb197..d55355c 100644
--- a/be/src/olap/rowset/segment_v2/rle_page.h
+++ b/be/src/olap/rowset/segment_v2/rle_page.h
@@ -23,6 +23,7 @@
#include "olap/rowset/segment_v2/common.h" // for rowid_t
#include "util/rle_encoding.h" // for RleEncoder/RleDecoder
#include "util/coding.h" // for encode_fixed32_le/decode_fixed32_le
+#include "util/owned_slice.h" // for owned_slice
namespace doris {
namespace segment_v2 {
@@ -97,13 +98,17 @@ public:
return Status::OK();
}
- Slice finish() override {
+ // this api will release the memory ownership of encoded data
+ // Note:
+ // reset() should be called after this function before reuse the builder
+ OwnedSlice finish() override {
_finished = true;
// here should Flush first and then encode the count header
// or it will lead to a bug if the header is less than 8 byte and the data is small
_rle_encoder->Flush();
encode_fixed32_le(&_buf[0], _count);
- return Slice(_buf.data(), _buf.size());
+ size_t buf_size = _buf.size();
+ return OwnedSlice(_buf.release(), buf_size);
}
void reset() override {
@@ -120,15 +125,6 @@ public:
return _rle_encoder->len();
}
- // this api will release the memory ownership of encoded data
- // Note:
- // release() should be called after finish
- // reset() should be called after this function before reuse the builder
- void release() override {
- uint8_t* ret = _buf.release();
- (void)ret;
- }
-
private:
typedef typename TypeTraits<Type>::CppType CppType;
enum {
diff --git a/be/src/util/owned_slice.h b/be/src/util/owned_slice.h
new file mode 100644
index 0000000..a17fae4
--- /dev/null
+++ b/be/src/util/owned_slice.h
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "gutil/macros.h"
+#include "util/slice.h"
+
+/**
+ * used for maintain slice' life cycle
+ * Attention!!! the input slice must own a data（char*） which using new to allocate memory,or will cause error
+ */
+
+namespace doris {
+
+/**
+ * originally make this class pageBuilder',to maintain the life cycle of returned slice
+ * Attention!!!
+ * 1. the input slice must own a data（char*） which using new to allocate memory,or will cause error
+ * 2. don't use std::move for this class
+ *
+ * use case:
+ * fastring buf;
+ * buf.append();
+ * size_t size = buf.size();
+ * Slice slice(buf.release(), size); // here is the key point, using buf.release;use buf.data() is error
+ *
+ * OwnedSlice owned_slice(slice);
+ */
+class OwnedSlice {
+
+public:
+ OwnedSlice() : _slice((uint8_t*)nullptr, 0) {}
+
+ explicit OwnedSlice(Slice slice) : _slice(slice) {}
+
+ OwnedSlice(uint8_t* _data, size_t size) :_slice(_data, size) {}
+
+ ~OwnedSlice(){
+ delete[] _slice.data;
+ }
+
+ OwnedSlice(OwnedSlice&& src) {
+ _slice.data = src._slice.data;
+ _slice.size = src._slice.size;
+ src._slice.data = nullptr;
+ src._slice.size = 0;
+ }
+
+ OwnedSlice& operator = (OwnedSlice&& src) {
+ if (this != &src) {
+ std::swap(_slice, src._slice);
+ }
+ return *this;
+ }
+
+ const Slice& slice() const {
+ return _slice;
+ }
+
+private:
+ DISALLOW_COPY_AND_ASSIGN(OwnedSlice);
+ Slice _slice;
+};
+
+}// namespace doris
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp b/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp
index fb6b2e5..16f2877 100644
--- a/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp
@@ -46,17 +46,17 @@ public:
const Slice *ptr = &slices[0];
Status ret = page_builder.add(reinterpret_cast<const uint8_t *>(ptr), &count);
- Slice s = page_builder.finish();
+ OwnedSlice s = page_builder.finish();
ASSERT_EQ(slices.size(), page_builder.count());
ASSERT_FALSE(page_builder.is_page_full());
// construct dict page
- Slice dict_slice;
+ OwnedSlice dict_slice;
Status status = page_builder.get_dictionary_page(&dict_slice);
ASSERT_TRUE(status.ok());
PageDecoderOptions dict_decoder_options;
std::unique_ptr<BinaryPlainPageDecoder> dict_page_decoder(
- new BinaryPlainPageDecoder(dict_slice, dict_decoder_options));
+ new BinaryPlainPageDecoder(dict_slice.slice(), dict_decoder_options));
status = dict_page_decoder->init();
ASSERT_TRUE(status.ok());
// because every slice is unique
@@ -64,7 +64,7 @@ public:
// decode
PageDecoderOptions decoder_options;
- BinaryDictPageDecoder page_decoder(s, decoder_options);
+ BinaryDictPageDecoder page_decoder(s.slice(), decoder_options);
page_decoder.set_dict_decoder(dict_page_decoder.get());
status = page_decoder.init();
@@ -110,7 +110,7 @@ public:
options.dict_page_size = 1 * 1024 * 1024;
BinaryDictPageBuilder page_builder(options);
size_t count = contents.size();
- std::vector<Slice> results;
+ std::vector<OwnedSlice> results;
std::vector<size_t> page_start_ids;
size_t total_size = 0;
page_start_ids.push_back(0);
@@ -119,29 +119,27 @@ public:
const Slice* ptr = &contents[i];
Status ret = page_builder.add(reinterpret_cast<const uint8_t *>(ptr), &add_num);
if (page_builder.is_page_full()) {
- Slice s = page_builder.finish();
- total_size += s.size;
- results.emplace_back(s);
- page_builder.release();
+ OwnedSlice s = page_builder.finish();
+ total_size += s.slice().size;
+ results.emplace_back(std::move(s));
page_builder.reset();
page_start_ids.push_back(i + 1);
}
i += add_num;
}
- Slice s = page_builder.finish();
- total_size += s.size;
- results.emplace_back(s);
- page_builder.release();
+ OwnedSlice s = page_builder.finish();
+ total_size += s.slice().size;
+ results.emplace_back(std::move(s));
page_start_ids.push_back(count);
- Slice dict_slice;
+ OwnedSlice dict_slice;
Status status = page_builder.get_dictionary_page(&dict_slice);
size_t data_size = total_size;
- total_size += dict_slice.size;
+ total_size += dict_slice.slice().size;
ASSERT_TRUE(status.ok());
LOG(INFO) << "total size:" << total_size << ", data size:" << data_size
- << ", dict size:" << dict_slice.size
+ << ", dict size:" << dict_slice.slice().size
<< " result page size:" << results.size();
// validate
@@ -152,13 +150,13 @@ public:
//int slice_index = 1;
PageDecoderOptions dict_decoder_options;
std::unique_ptr<BinaryPlainPageDecoder> dict_page_decoder(
- new BinaryPlainPageDecoder(dict_slice, dict_decoder_options));
+ new BinaryPlainPageDecoder(dict_slice.slice(), dict_decoder_options));
status = dict_page_decoder->init();
ASSERT_TRUE(status.ok());
// decode
PageDecoderOptions decoder_options;
- BinaryDictPageDecoder page_decoder(results[slice_index], decoder_options);
+ BinaryDictPageDecoder page_decoder(results[slice_index].slice(), decoder_options);
status = page_decoder.init();
page_decoder.set_dict_decoder(dict_page_decoder.get());
ASSERT_TRUE(status.ok());
diff --git a/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp b/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp
index b93964b..1947f79 100644
--- a/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp
@@ -52,10 +52,10 @@ public:
Slice *ptr = &slices[0];
Status ret = page_builder.add(reinterpret_cast<const uint8_t *>(ptr), &count);
-
- Slice s = page_builder.finish();
+
+ OwnedSlice owned_slice = page_builder.finish();
PageDecoderOptions decoder_options;
- PageDecoderType page_decoder(s, decoder_options);
+ PageDecoderType page_decoder(owned_slice.slice(), decoder_options);
Status status = page_decoder.init();
ASSERT_TRUE(status.ok());
diff --git a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
index aefeee6..202f57c 100644
--- a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
@@ -56,12 +56,12 @@ public:
PageBuilderType page_builder(options);
page_builder.add(reinterpret_cast<const uint8_t *>(src), &size);
- Slice s = page_builder.finish();
- LOG(INFO) << "RLE Encoded size for 10k values: " << s.size
+ OwnedSlice s = page_builder.finish();
+ LOG(INFO) << "RLE Encoded size for 10k values: " << s.slice().size
<< ", original size:" << size * sizeof(CppType);
segment_v2::PageDecoderOptions decoder_options;
- PageDecoderType page_decoder(s, decoder_options);
+ PageDecoderType page_decoder(s.slice(), decoder_options);
Status status = page_decoder.init();
ASSERT_TRUE(status.ok());
ASSERT_EQ(0, page_decoder.current_index());
diff --git a/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp b/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp
index 3a80e1c..9de21f2 100644
--- a/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp
@@ -45,8 +45,8 @@ public:
builder.add(nullptr, 1);
}
builder.flush();
- Slice zone_map_page = builder.finish();
- ColumnZoneMap column_zone_map(zone_map_page);
+ OwnedSlice zone_map_page = builder.finish();
+ ColumnZoneMap column_zone_map(zone_map_page.slice());
Status status = column_zone_map.load();
ASSERT_TRUE(status.ok());
ASSERT_EQ(3, column_zone_map.num_pages());
@@ -88,8 +88,8 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) {
builder.add(nullptr, 1);
}
builder.flush();
- Slice zone_map_page = builder.finish();
- ColumnZoneMap column_zone_map(zone_map_page);
+ OwnedSlice zone_map_page = builder.finish();
+ ColumnZoneMap column_zone_map(zone_map_page.slice());
Status status = column_zone_map.load();
ASSERT_TRUE(status.ok());
ASSERT_EQ(3, column_zone_map.num_pages());
diff --git a/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp b/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
index 6806408..00a9803 100644
--- a/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
@@ -53,13 +53,13 @@ public:
builder_options.data_page_size = 256 * 1024;
PageBuilderType for_page_builder(builder_options);
for_page_builder.add(reinterpret_cast<const uint8_t *>(src), &size);
- Slice s = for_page_builder.finish();
+ OwnedSlice s = for_page_builder.finish();
ASSERT_EQ(size, for_page_builder.count());
- LOG(INFO) << "FrameOfReference Encoded size for 10k values: " << s.size
+ LOG(INFO) << "FrameOfReference Encoded size for 10k values: " << s.slice().size
<< ", original size:" << size * sizeof(CppType);
PageDecoderOptions decoder_options;
- PageDecoderType for_page_decoder(s, decoder_options);
+ PageDecoderType for_page_decoder(s.slice(), decoder_options);
Status status = for_page_decoder.init();
ASSERT_TRUE(status.ok());
ASSERT_EQ(0, for_page_decoder.current_index());
@@ -153,10 +153,10 @@ TEST_F(FrameOfReferencePageTest, TestInt32SequenceBlockEncoderSize) {
builder_options.data_page_size = 256 * 1024;
segment_v2::FrameOfReferencePageBuilder<OLAP_FIELD_TYPE_INT> page_builder(builder_options);
page_builder.add(reinterpret_cast<const uint8_t *>(ints.get()), &size);
- Slice s = page_builder.finish();
+ OwnedSlice s = page_builder.finish();
// body: 4 bytes min value + 128 * 1 /8 packing value = 20
// header: 1 + 1 + 4 = 6
- ASSERT_EQ(26, s.size);
+ ASSERT_EQ(26, s.slice().size);
}
TEST_F(FrameOfReferencePageTest, TestInt32NormalBlockEncoderSize) {
@@ -169,10 +169,10 @@ TEST_F(FrameOfReferencePageTest, TestInt32NormalBlockEncoderSize) {
builder_options.data_page_size = 256 * 1024;
segment_v2::FrameOfReferencePageBuilder<OLAP_FIELD_TYPE_INT> page_builder(builder_options);
page_builder.add(reinterpret_cast<const uint8_t *>(ints.get()), &size);
- Slice s = page_builder.finish();
+ OwnedSlice s = page_builder.finish();
// body: 4 bytes min value + 128 * 7 /8 packing value = 116
// header: 1 + 1 + 4 = 6
- ASSERT_EQ(122, s.size);
+ ASSERT_EQ(122, s.slice().size);
}
}
diff --git a/be/test/olap/rowset/segment_v2/plain_page_test.cpp b/be/test/olap/rowset/segment_v2/plain_page_test.cpp
index ea276a3..9e13c96 100644
--- a/be/test/olap/rowset/segment_v2/plain_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/plain_page_test.cpp
@@ -67,10 +67,10 @@ public:
PageBuilderType page_builder(options);
page_builder.add(reinterpret_cast<const uint8_t *>(src), &size);
- Slice s = page_builder.finish();
+ OwnedSlice s = page_builder.finish();
PageDecoderOptions decoder_options;
- PageDecoderType page_decoder(s, decoder_options);
+ PageDecoderType page_decoder(s.slice(), decoder_options);
Status status = page_decoder.init();
ASSERT_TRUE(status.ok());
diff --git a/be/test/olap/rowset/segment_v2/rle_page_test.cpp b/be/test/olap/rowset/segment_v2/rle_page_test.cpp
index efb6282..d548b6b 100644
--- a/be/test/olap/rowset/segment_v2/rle_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/rle_page_test.cpp
@@ -56,13 +56,13 @@ public:
builder_options.data_page_size = 256 * 1024;
PageBuilderType rle_page_builder(builder_options);
rle_page_builder.add(reinterpret_cast<const uint8_t *>(src), &size);
- Slice s = rle_page_builder.finish();
+ OwnedSlice s = rle_page_builder.finish();
ASSERT_EQ(size, rle_page_builder.count());
- LOG(INFO) << "RLE Encoded size for 10k values: " << s.size
+ LOG(INFO) << "RLE Encoded size for 10k values: " << s.slice().size
<< ", original size:" << size * sizeof(CppType);
PageDecoderOptions decodeder_options;
- PageDecoderType rle_page_decoder(s, decodeder_options);
+ PageDecoderType rle_page_decoder(s.slice(), decodeder_options);
Status status = rle_page_decoder.init();
ASSERT_TRUE(status.ok());
ASSERT_EQ(0, rle_page_decoder.current_index());
@@ -146,11 +146,11 @@ TEST_F(RlePageTest, TestRleInt32BlockEncoderSize) {
builder_options.data_page_size = 256 * 1024;
segment_v2::RlePageBuilder<OLAP_FIELD_TYPE_INT> rle_page_builder(builder_options);
rle_page_builder.add(reinterpret_cast<const uint8_t *>(ints.get()), &size);
- Slice s = rle_page_builder.finish();
+ OwnedSlice s = rle_page_builder.finish();
// 4 bytes header
// 2 bytes indicate_value(): 0x64 << 1 | 1 = 201
// 4 bytes values
- ASSERT_EQ(10, s.size);
+ ASSERT_EQ(10, s.slice().size);
}
TEST_F(RlePageTest, TestRleBoolBlockEncoderRandom) {
@@ -180,11 +180,11 @@ TEST_F(RlePageTest, TestRleBoolBlockEncoderSize) {
builder_options.data_page_size = 256 * 1024;
segment_v2::RlePageBuilder<OLAP_FIELD_TYPE_BOOL> rle_page_builder(builder_options);
rle_page_builder.add(reinterpret_cast<const uint8_t *>(bools.get()), &size);
- Slice s = rle_page_builder.finish();
+ OwnedSlice s = rle_page_builder.finish();
// 4 bytes header
// 2 bytes indicate_value(): 0x64 << 1 | 1 = 201
// 1 bytes values
- ASSERT_EQ(7, s.size);
+ ASSERT_EQ(7, s.slice().size);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org