You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2020/02/03 16:16:14 UTC
[incubator-doris] branch master updated: Add file cache for v2
(#2782)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a27e890 Add file cache for v2 (#2782)
a27e890 is described below
commit a27e89065b692acd72ef2a1a5d616621923a3da3
Author: kangpinghuang <kp...@163.com>
AuthorDate: Tue Feb 4 00:16:01 2020 +0800
Add file cache for v2 (#2782)
Add file descriptor cache for segment v2 to solve too many open file problems
---
be/src/env/env_posix.cpp | 3 +-
.../olap/rowset/segment_v2/bitmap_index_reader.cpp | 4 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 8 +-
.../segment_v2/bloom_filter_index_reader.cpp | 2 +-
.../rowset/segment_v2/bloom_filter_index_reader.h | 6 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 29 +++--
be/src/olap/rowset/segment_v2/column_reader.h | 22 +++-
.../rowset/segment_v2/indexed_column_reader.cpp | 16 ++-
.../olap/rowset/segment_v2/indexed_column_reader.h | 23 +++-
be/src/olap/rowset/segment_v2/segment.cpp | 19 ++-
be/src/olap/rowset/segment_v2/segment.h | 2 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 4 +
be/src/olap/rowset/segment_v2/segment_iterator.h | 4 +
be/src/util/CMakeLists.txt | 4 +-
be/src/util/file_cache.cpp | 54 +++++++++
be/src/util/file_cache.h | 129 +++++++++++++++++++++
be/src/util/file_manager.cpp | 38 ++++++
be/src/util/file_manager.h | 54 +++++++++
.../bloom_filter_index_reader_writer_test.cpp | 6 +-
.../segment_v2/column_reader_writer_test.cpp | 10 +-
.../segment_v2/index_column_reader_writer_test.cpp | 23 +---
be/test/util/CMakeLists.txt | 3 +-
be/test/util/file_cache_test.cpp | 71 ++++++++++++
be/test/util/file_manager_test.cpp | 67 +++++++++++
run-ut.sh | 2 +
25 files changed, 533 insertions(+), 70 deletions(-)
diff --git a/be/src/env/env_posix.cpp b/be/src/env/env_posix.cpp
index 578991a..3c6e632 100644
--- a/be/src/env/env_posix.cpp
+++ b/be/src/env/env_posix.cpp
@@ -25,6 +25,7 @@
#include "gutil/strings/substitute.h"
#include "util/errno.h"
#include "util/slice.h"
+#include "util/file_cache.h"
namespace doris {
@@ -484,7 +485,6 @@ private:
class PosixEnv : public Env {
public:
- PosixEnv() { }
~PosixEnv() override { }
Status new_sequential_file(
@@ -498,6 +498,7 @@ public:
return Status::OK();
}
+ // get a RandomAccessFile pointer without file cache
Status new_random_access_file(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result) override {
return new_random_access_file(RandomAccessFileOptions(), fname, result);
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
index 17d6aa9..3a77158 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
@@ -27,8 +27,8 @@ Status BitmapIndexReader::load() {
const IndexedColumnMetaPB& bitmap_meta = _bitmap_index_meta.bitmap_column();
_has_null = _bitmap_index_meta.has_null();
- _dict_column_reader.reset(new IndexedColumnReader(_file, dict_meta));
- _bitmap_column_reader.reset(new IndexedColumnReader(_file, bitmap_meta));
+ _dict_column_reader.reset(new IndexedColumnReader(_file_name, dict_meta));
+ _bitmap_column_reader.reset(new IndexedColumnReader(_file_name, bitmap_meta));
RETURN_IF_ERROR(_dict_column_reader->load());
RETURN_IF_ERROR(_bitmap_column_reader->load());
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
index 18f4f01..10a6d22 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
@@ -40,9 +40,9 @@ class IndexedColumnIterator;
class BitmapIndexReader {
public:
- explicit BitmapIndexReader(RandomAccessFile* file,
+ explicit BitmapIndexReader(const std::string& file_name,
const BitmapIndexColumnPB& bitmap_index_meta)
- : _file(file),
+ : _file_name(file_name),
_bitmap_index_meta(bitmap_index_meta){
_typeinfo = get_type_info(OLAP_FIELD_TYPE_VARCHAR);
}
@@ -63,7 +63,7 @@ public:
private:
friend class BitmapIndexIterator;
- RandomAccessFile* _file;
+ std::string _file_name;
const TypeInfo* _typeinfo;
const BitmapIndexColumnPB& _bitmap_index_meta;
bool _has_null = false;
@@ -78,7 +78,7 @@ public:
_dict_column_iter(reader->_dict_column_reader.get()),
_bitmap_column_iter(reader->_bitmap_column_reader.get()),
_current_rowid(0),
- _pool(new MemPool(&_tracker)){
+ _pool(new MemPool(&_tracker)) {
}
bool has_null_bitmap() const { return _reader->_has_null; }
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index 6f0c293..09a54e2 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -26,7 +26,7 @@ namespace segment_v2 {
Status BloomFilterIndexReader::load() {
const IndexedColumnMetaPB& bf_index_meta = _bloom_filter_index_meta.bloom_filter();
- _bloom_filter_reader.reset(new IndexedColumnReader(_file, bf_index_meta));
+ _bloom_filter_reader.reset(new IndexedColumnReader(_file_name, bf_index_meta));
RETURN_IF_ERROR(_bloom_filter_reader->load());
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
index ffa9e7b..c9bfaae 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
@@ -43,9 +43,9 @@ class BloomFilter;
class BloomFilterIndexReader {
public:
- explicit BloomFilterIndexReader(RandomAccessFile* file,
+ explicit BloomFilterIndexReader(const std::string& file_name,
const BloomFilterIndexPB& bloom_filter_index_meta)
- : _file(file),
+ : _file_name(file_name),
_bloom_filter_index_meta(bloom_filter_index_meta) {
_typeinfo = get_type_info(OLAP_FIELD_TYPE_VARCHAR);
}
@@ -62,7 +62,7 @@ public:
private:
friend class BloomFilterIndexIterator;
- RandomAccessFile* _file;
+ std::string _file_name;
const TypeInfo* _typeinfo;
BloomFilterIndexPB _bloom_filter_index_meta;
std::unique_ptr<IndexedColumnReader> _bloom_filter_reader;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 99e4dc7..61fb2fd 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -33,6 +33,7 @@
#include "util/crc32c.h"
#include "util/rle_encoding.h" // for RleDecoder
#include "util/block_compression.h"
+#include "util/file_manager.h"
#include "olap/rowset/segment_v2/binary_dict_page.h" // for BinaryDictPageDecoder
#include "olap/rowset/segment_v2/bloom_filter_index_reader.h"
@@ -44,10 +45,10 @@ using strings::Substitute;
Status ColumnReader::create(const ColumnReaderOptions& opts,
const ColumnMetaPB& meta,
uint64_t num_rows,
- RandomAccessFile* file,
+ const std::string& file_name,
std::unique_ptr<ColumnReader>* reader) {
std::unique_ptr<ColumnReader> reader_local(
- new ColumnReader(opts, meta, num_rows, file));
+ new ColumnReader(opts, meta, num_rows, file_name));
RETURN_IF_ERROR(reader_local->init());
*reader = std::move(reader_local);
return Status::OK();
@@ -56,8 +57,8 @@ Status ColumnReader::create(const ColumnReaderOptions& opts,
ColumnReader::ColumnReader(const ColumnReaderOptions& opts,
const ColumnMetaPB& meta,
uint64_t num_rows,
- RandomAccessFile* file)
- : _opts(opts), _meta(meta), _num_rows(num_rows), _file(file) {
+ const std::string& file_name)
+ : _opts(opts), _meta(meta), _num_rows(num_rows), _file_name(file_name) {
}
ColumnReader::~ColumnReader() = default;
@@ -84,10 +85,18 @@ Status ColumnReader::new_bitmap_index_iterator(BitmapIndexIterator** iterator) {
}
Status ColumnReader::read_page(const PagePointer& pp, OlapReaderStatistics* stats, PageHandle* handle) {
+ OpenedFileHandle<RandomAccessFile> file_handle;
+ RETURN_IF_ERROR(FileManager::instance()->open_file(_file_name, &file_handle));
+ RandomAccessFile* input_file = file_handle.file();
+ return read_page(input_file, pp, stats, handle);
+}
+
+Status ColumnReader::read_page(RandomAccessFile* file, const PagePointer& pp,
+ OlapReaderStatistics* stats, PageHandle* handle) {
stats->total_pages_num++;
auto cache = StoragePageCache::instance();
PageCacheHandle cache_handle;
- StoragePageCache::CacheKey cache_key(_file->file_name(), pp.offset);
+ StoragePageCache::CacheKey cache_key(file->file_name(), pp.offset);
if (cache->lookup(cache_key, &cache_handle)) {
// we find page in cache, use it
*handle = PageHandle(std::move(cache_handle));
@@ -106,7 +115,7 @@ Status ColumnReader::read_page(const PagePointer& pp, OlapReaderStatistics* stat
Slice page_slice(page.get(), page_size);
{
SCOPED_RAW_TIMER(&stats->io_ns);
- RETURN_IF_ERROR(_file->read_at(pp.offset, page_slice));
+ RETURN_IF_ERROR(file->read_at(pp.offset, page_slice));
stats->compressed_bytes_read += page_size;
}
@@ -275,7 +284,7 @@ Status ColumnReader::_load_zone_map_index() {
Status ColumnReader::_load_bitmap_index() {
if (_meta.has_bitmap_index()) {
const BitmapIndexColumnPB& bitmap_index_meta = _meta.bitmap_index();
- _bitmap_index_reader.reset(new BitmapIndexReader(_file, bitmap_index_meta));
+ _bitmap_index_reader.reset(new BitmapIndexReader(_file_name, bitmap_index_meta));
RETURN_IF_ERROR(_bitmap_index_reader->load());
} else {
_bitmap_index_reader.reset(nullptr);
@@ -286,7 +295,7 @@ Status ColumnReader::_load_bitmap_index() {
Status ColumnReader::_load_bloom_filter_index() {
if (_meta.has_bloom_filter_index()) {
const BloomFilterIndexPB& bloom_filter_index_meta = _meta.bloom_filter_index();
- _bloom_filter_index_reader.reset(new BloomFilterIndexReader(_file, bloom_filter_index_meta));
+ _bloom_filter_index_reader.reset(new BloomFilterIndexReader(_file_name, bloom_filter_index_meta));
RETURN_IF_ERROR(_bloom_filter_index_reader->load());
} else {
_bloom_filter_index_reader.reset(nullptr);
@@ -454,7 +463,7 @@ Status FileColumnIterator::_load_next_page(bool* eos) {
// it ready to read
Status FileColumnIterator::_read_page(const OrdinalPageIndexIterator& iter, ParsedPage* page) {
page->page_pointer = iter.page();
- RETURN_IF_ERROR(_reader->read_page(page->page_pointer, _opts.stats, &page->page_handle));
+ RETURN_IF_ERROR(_reader->read_page(_file, page->page_pointer, _opts.stats, &page->page_handle));
// TODO(zc): read page from file
Slice data = page->page_handle.data();
@@ -496,7 +505,7 @@ Status FileColumnIterator::_read_page(const OrdinalPageIndexIterator& iter, Pars
if (binary_dict_page_decoder->is_dict_encoding()) {
if (_dict_decoder == nullptr) {
PagePointer pp = _reader->get_dict_page_pointer();
- RETURN_IF_ERROR(_reader->read_page(pp, _opts.stats, &_dict_page_handle));
+ RETURN_IF_ERROR(_reader->read_page(_file, pp, _opts.stats, &_dict_page_handle));
_dict_decoder.reset(new BinaryPlainPageDecoder(_dict_page_handle.data()));
RETURN_IF_ERROR(_dict_decoder->init());
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 9bf66c3..1590005 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -33,6 +33,7 @@
#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle
#include "olap/rowset/segment_v2/parsed_page.h" // for ParsedPage
#include "util/once.h"
+#include "util/file_cache.h"
namespace doris {
@@ -57,6 +58,7 @@ struct ColumnReaderOptions {
struct ColumnIteratorOptions {
// reader statistics
OlapReaderStatistics* stats = nullptr;
+ RandomAccessFile* file = nullptr;
};
// There will be concurrent users to read the same column. So
@@ -70,7 +72,7 @@ public:
static Status create(const ColumnReaderOptions& opts,
const ColumnMetaPB& meta,
uint64_t num_rows,
- RandomAccessFile* file,
+ const std::string& file_name,
std::unique_ptr<ColumnReader>* reader);
~ColumnReader();
@@ -85,8 +87,13 @@ public:
Status seek_at_or_before(rowid_t rowid, OrdinalPageIndexIterator* iter);
// read a page from file into a page handle
+ // use reader owned _file(usually is Descriptor<RandomAccessFile>*) to read page
Status read_page(const PagePointer& pp, OlapReaderStatistics* stats, PageHandle* handle);
+ // read a page from file into a page handle
+ // use file(usually is RandomAccessFile*) to read page
+ Status read_page(RandomAccessFile* file, const PagePointer& pp, OlapReaderStatistics* stats, PageHandle* handle);
+
bool is_nullable() const { return _meta.is_nullable(); }
const EncodingInfo* encoding_info() const { return _encoding_info; }
@@ -120,7 +127,7 @@ private:
ColumnReader(const ColumnReaderOptions& opts,
const ColumnMetaPB& meta,
uint64_t num_rows,
- RandomAccessFile* file);
+ const std::string& file_name);
Status init();
// Read and load necessary column indexes into memory if it hasn't been loaded.
@@ -151,7 +158,7 @@ private:
ColumnReaderOptions _opts;
ColumnMetaPB _meta;
uint64_t _num_rows;
- RandomAccessFile* _file;
+ std::string _file_name;
// initialized in init()
const TypeInfo* _type_info = nullptr;
@@ -228,6 +235,13 @@ public:
FileColumnIterator(ColumnReader* reader);
~FileColumnIterator() override;
+ Status init(const ColumnIteratorOptions& opts) override {
+ RETURN_IF_ERROR(ColumnIterator::init(opts));
+ DCHECK(_opts.file != nullptr);
+ _file = _opts.file;
+ return Status::OK();
+ }
+
Status seek_to_first() override;
Status seek_to_ordinal(rowid_t ord_idx) override;
@@ -274,6 +288,8 @@ private:
// page indexes those are DEL_PARTIAL_SATISFIED
std::vector<uint32_t> _delete_partial_statisfied_pages;
+
+ RandomAccessFile* _file;
};
// This iterator is used to read default value column
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index c4ad315..d40182a 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -27,6 +27,7 @@
#include "olap/rowset/segment_v2/page_decoder.h" // for PagePointer
#include "util/crc32c.h"
#include "util/rle_encoding.h" // for RleDecoder
+#include "util/file_manager.h"
namespace doris {
namespace segment_v2 {
@@ -42,12 +43,15 @@ Status IndexedColumnReader::load() {
RETURN_IF_ERROR(get_block_compression_codec(_meta.compression(), &_compress_codec));
_validx_key_coder = get_key_coder(_type_info->type());
+ OpenedFileHandle<RandomAccessFile> file_handle;
+ RETURN_IF_ERROR(FileManager::instance()->open_file(_file_name, &file_handle));
+ RandomAccessFile* input_file = file_handle.file();
// read and parse ordinal index page when exists
if (_meta.has_ordinal_index_meta()) {
if (_meta.ordinal_index_meta().is_root_data_page()) {
_sole_data_page = PagePointer(_meta.ordinal_index_meta().root_page());
} else {
- RETURN_IF_ERROR(read_page(_meta.ordinal_index_meta().root_page(), &_ordinal_index_page_handle));
+ RETURN_IF_ERROR(read_page(input_file, _meta.ordinal_index_meta().root_page(), &_ordinal_index_page_handle));
RETURN_IF_ERROR(_ordinal_index_reader.parse(_ordinal_index_page_handle.data()));
_has_index_page = true;
}
@@ -58,7 +62,7 @@ Status IndexedColumnReader::load() {
if (_meta.value_index_meta().is_root_data_page()) {
_sole_data_page = PagePointer(_meta.value_index_meta().root_page());
} else {
- RETURN_IF_ERROR(read_page(_meta.value_index_meta().root_page(), &_value_index_page_handle));
+ RETURN_IF_ERROR(read_page(input_file, _meta.value_index_meta().root_page(), &_value_index_page_handle));
RETURN_IF_ERROR(_value_index_reader.parse(_value_index_page_handle.data()));
_has_index_page = true;
}
@@ -67,10 +71,10 @@ Status IndexedColumnReader::load() {
return Status::OK();
}
-Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle) const {
+Status IndexedColumnReader::read_page(RandomAccessFile* file, const PagePointer& pp, PageHandle* handle) const {
auto cache = StoragePageCache::instance();
PageCacheHandle cache_handle;
- StoragePageCache::CacheKey cache_key(_file->file_name(), pp.offset);
+ StoragePageCache::CacheKey cache_key(file->file_name(), pp.offset);
if (cache->lookup(cache_key, &cache_handle)) {
// we find page in cache, use it
*handle = PageHandle(std::move(cache_handle));
@@ -86,7 +90,7 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle)
// this buffer will assigned uncompressed page, and origin content will be freed.
std::unique_ptr<uint8_t[]> page(new uint8_t[page_size]);
Slice page_slice(page.get(), page_size);
- RETURN_IF_ERROR(_file->read_at(pp.offset, page_slice));
+ RETURN_IF_ERROR(file->read_at(pp.offset, page_slice));
size_t data_size = page_size - 4;
if (_verify_checksum) {
@@ -124,7 +128,7 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle)
///////////////////////////////////////////////////////////////////////////////
Status IndexedColumnIterator::_read_data_page(const PagePointer& page_pointer, ParsedPage* page) {
- RETURN_IF_ERROR(_reader->read_page(page_pointer, &page->page_handle));
+ RETURN_IF_ERROR(_reader->read_page(_file, page_pointer, &page->page_handle));
Slice data = page->page_handle.data();
// decode first rowid
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index a1bff17..f5bb4d3 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -29,6 +29,8 @@
#include "olap/rowset/segment_v2/parsed_page.h"
#include "util/block_compression.h"
#include "util/slice.h"
+#include "util/file_cache.h"
+#include "util/file_manager.h"
namespace doris {
@@ -44,12 +46,14 @@ class IndexedColumnIterator;
// thread-safe reader for IndexedColumn (see comments of `IndexedColumnWriter` to understand what IndexedColumn is)
class IndexedColumnReader {
public:
- explicit IndexedColumnReader(RandomAccessFile* file, const IndexedColumnMetaPB& meta)
- : _file(file), _meta(meta) {};
+ explicit IndexedColumnReader(const std::string& file_name, const IndexedColumnMetaPB& meta)
+ : _file_name(file_name), _meta(meta) {};
Status load();
- Status read_page(const PagePointer& pp, PageHandle* handle) const;
+ // read a page from file into a page handle
+ // use file(usually is RandomAccessFile*) to read page
+ Status read_page(RandomAccessFile* file, const PagePointer& pp, PageHandle* handle) const;
int64_t num_values() const { return _num_values; }
@@ -64,7 +68,7 @@ public:
private:
friend class IndexedColumnIterator;
- RandomAccessFile* _file;
+ std::string _file_name;
IndexedColumnMetaPB _meta;
int64_t _num_values = 0;
// whether this column contains any index page.
@@ -89,7 +93,12 @@ public:
explicit IndexedColumnIterator(const IndexedColumnReader* reader)
: _reader(reader),
_ordinal_iter(&reader->_ordinal_index_reader),
- _value_iter(&reader->_value_index_reader) {
+ _value_iter(&reader->_value_index_reader),
+ _file(nullptr) {
+ auto st = FileManager::instance()->open_file(_reader->_file_name, &_file_handle);
+ DCHECK(st.ok());
+ WARN_IF_ERROR(st, "open file failed:" + _reader->_file_name);
+ _file = _file_handle.file();
}
// Seek to the given ordinal entry. Entry 0 is the first entry.
@@ -131,6 +140,10 @@ private:
std::unique_ptr<ParsedPage> _data_page;
// next_batch() will read from this position
rowid_t _current_rowid = 0;
+ // open file handle
+ OpenedFileHandle<RandomAccessFile> _file_handle;
+ // file to read
+ RandomAccessFile* _file;
};
} // namespace segment_v2
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 6adb40d..dc1cab8 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -27,6 +27,7 @@
#include "util/slice.h" // Slice
#include "olap/tablet_schema.h"
#include "util/crc32c.h"
+#include "util/file_manager.h"
namespace doris {
namespace segment_v2 {
@@ -54,7 +55,6 @@ Segment::Segment(
Segment::~Segment() = default;
Status Segment::_open() {
- RETURN_IF_ERROR(Env::Default()->new_random_access_file(_fname, &_input_file));
RETURN_IF_ERROR(_parse_footer());
RETURN_IF_ERROR(_create_column_readers());
return Status::OK();
@@ -118,15 +118,18 @@ Status Segment::new_iterator(const Schema& schema,
Status Segment::_parse_footer() {
// Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4), MagicNumber(4)
+ OpenedFileHandle<RandomAccessFile> file_handle;
+ RETURN_IF_ERROR(FileManager::instance()->open_file(_fname, &file_handle));
+ RandomAccessFile* input_file = file_handle.file();
uint64_t file_size;
- RETURN_IF_ERROR(_input_file->size(&file_size));
+ RETURN_IF_ERROR(input_file->size(&file_size));
if (file_size < 12) {
return Status::Corruption(Substitute("Bad segment file $0: file size $1 < 12", _fname, file_size));
}
uint8_t fixed_buf[12];
- RETURN_IF_ERROR(_input_file->read_at(file_size - 12, Slice(fixed_buf, 12)));
+ RETURN_IF_ERROR(input_file->read_at(file_size - 12, Slice(fixed_buf, 12)));
// validate magic number
if (memcmp(fixed_buf + 8, k_segment_magic, k_segment_magic_length) != 0) {
@@ -141,7 +144,7 @@ Status Segment::_parse_footer() {
}
std::string footer_buf;
footer_buf.resize(footer_length);
- RETURN_IF_ERROR(_input_file->read_at(file_size - 12 - footer_length, footer_buf));
+ RETURN_IF_ERROR(input_file->read_at(file_size - 12 - footer_length, footer_buf));
// validate footer PB's checksum
uint32_t expect_checksum = decode_fixed32_le(fixed_buf + 4);
@@ -162,9 +165,12 @@ Status Segment::_parse_footer() {
Status Segment::_load_index() {
return _load_index_once.call([this] {
// read short key index content
+ OpenedFileHandle<RandomAccessFile> file_handle;
+ RETURN_IF_ERROR(FileManager::instance()->open_file(_fname, &file_handle));
+ RandomAccessFile* input_file = file_handle.file();
_sk_index_buf.resize(_footer.short_key_index_page().size());
Slice slice(_sk_index_buf.data(), _sk_index_buf.size());
- RETURN_IF_ERROR(_input_file->read_at(_footer.short_key_index_page().offset(), slice));
+ RETURN_IF_ERROR(input_file->read_at(_footer.short_key_index_page().offset(), slice));
// Parse short key index
_sk_index_decoder.reset(new ShortKeyIndexDecoder(_sk_index_buf));
@@ -189,8 +195,9 @@ Status Segment::_create_column_readers() {
ColumnReaderOptions opts;
std::unique_ptr<ColumnReader> reader;
+ // pass Descriptor<RandomAccessFile>* to column reader
RETURN_IF_ERROR(ColumnReader::create(
- opts, _footer.columns(iter->second), _footer.num_rows(), _input_file.get(), &reader));
+ opts, _footer.columns(iter->second), _footer.num_rows(), _fname, &reader));
_column_readers[ordinal] = std::move(reader);
}
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index b3fdc0c..831d8c8 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -122,12 +122,12 @@ private:
Status _load_index();
private:
+ friend class SegmentIterator;
std::string _fname;
uint32_t _segment_id;
const TabletSchema* _tablet_schema;
SegmentFooterPB _footer;
- std::unique_ptr<RandomAccessFile> _input_file;
// Map from column unique id to column ordinal in footer's ColumnMetaPB
// If we can't find unique id from it, it means this segment is created
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 551afc3..6f0f347 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -124,6 +124,8 @@ Status SegmentIterator::init(const StorageReadOptions& opts) {
Status SegmentIterator::_init() {
DorisMetrics::segment_read_total.increment(1);
+ // get file handle from file descriptor of segment
+ RETURN_IF_ERROR(FileManager::instance()->open_file(_segment->_fname, &_file_handle));
_row_bitmap.addRange(0, _segment->num_rows());
RETURN_IF_ERROR(_init_return_column_iterators());
RETURN_IF_ERROR(_init_bitmap_index_iterators());
@@ -195,6 +197,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra
RETURN_IF_ERROR(_segment->new_column_iterator(cid, &_column_iterators[cid]));
ColumnIteratorOptions iter_opts;
iter_opts.stats = _opts.stats;
+ iter_opts.file = _file_handle.file();
RETURN_IF_ERROR(_column_iterators[cid]->init(iter_opts));
}
}
@@ -312,6 +315,7 @@ Status SegmentIterator::_init_return_column_iterators() {
RETURN_IF_ERROR(_segment->new_column_iterator(cid, &_column_iterators[cid]));
ColumnIteratorOptions iter_opts;
iter_opts.stats = _opts.stats;
+ iter_opts.file = _file_handle.file();
RETURN_IF_ERROR(_column_iterators[cid]->init(iter_opts));
}
}
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index fe024c4..b3a9310 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -30,6 +30,7 @@
#include "olap/rowset/segment_v2/column_zone_map.h"
#include "olap/rowset/segment_v2/ordinal_page_index.h"
#include "olap/olap_cond.h"
+#include "util/file_cache.h"
namespace doris {
@@ -120,6 +121,9 @@ private:
// used to binary search the rowid for a given key
// only used in `_get_row_ranges_by_keys`
std::unique_ptr<RowBlockV2> _seek_block;
+
+ // Handle for file to read
+ OpenedFileHandle<RandomAccessFile> _file_handle;
};
}
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index 2159056..ec60a61 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -83,10 +83,12 @@ set(UTIL_FILES
frame_of_reference_coding.cpp
minizip/ioapi.c
minizip/unzip.c
- zip_util.cpp
+ zip_util.cpp
utf8_check.cpp
cgroup_util.cpp
path_util.cpp
+ file_manager.cpp
+ file_cache.cpp
)
if (WITH_MYSQL)
diff --git a/be/src/util/file_cache.cpp b/be/src/util/file_cache.cpp
new file mode 100644
index 0000000..d7085b7
--- /dev/null
+++ b/be/src/util/file_cache.cpp
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/file_cache.h"
+
+#include "gutil/strings/substitute.h"
+#include "env/env.h"
+
+namespace doris {
+
+template <class FileType>
+FileCache<FileType>::FileCache(const std::string& cache_name, int max_open_files) :
+ _cache_name(cache_name),
+ _cache(new_lru_cache(max_open_files)) { }
+
+template <class FileType>
+bool FileCache<FileType>::lookup(const std::string& file_name, OpenedFileHandle<FileType>* file_handle) {
+ CacheKey key(file_name);
+ auto lru_handle = _cache->lookup(key);
+ if (lru_handle == nullptr) {
+ return false;
+ }
+ *file_handle = OpenedFileHandle<FileType>(_cache.get(), lru_handle);
+ return true;
+}
+
+template <class FileType>
+void FileCache<FileType>::insert(const std::string& file_name, FileType* file, OpenedFileHandle<FileType>* file_handle) {
+ auto deleter = [](const CacheKey& key, void* value) {
+ delete (FileType*)value;
+ };
+ CacheKey key(file_name);
+ auto lru_handle = _cache->insert(key, file, 1, deleter);
+ *file_handle = OpenedFileHandle<FileType>(_cache.get(), lru_handle);
+}
+
+// Explicit specialization for callers outside this compilation unit.
+template class FileCache<RandomAccessFile>;
+
+} // namespace doris
diff --git a/be/src/util/file_cache.h b/be/src/util/file_cache.h
new file mode 100644
index 0000000..d7370c1
--- /dev/null
+++ b/be/src/util/file_cache.h
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "olap/lru_cache.h"
+
+namespace doris {
+
+class Env;
+
+// A "smart" retrieved LRU cache handle.
+//
+// The cache handle is released when this object goes out of scope, possibly
+// closing the opened file if it is no longer in the cache.
+template <class FileType>
+class OpenedFileHandle {
+public:
+ OpenedFileHandle() : _cache(nullptr), _handle(nullptr) { }
+
+ // A opened file handle
+ explicit OpenedFileHandle(Cache* cache, Cache::Handle* handle)
+ : _cache(cache), _handle(handle) { }
+
+ // release cache handle
+ ~OpenedFileHandle() {
+ if (_handle != nullptr) {
+ _cache->release(_handle);
+ }
+ }
+
+ OpenedFileHandle(OpenedFileHandle&& other) noexcept {
+ std::swap(_cache, other._cache);
+ std::swap(_handle, other._handle);
+ }
+
+ OpenedFileHandle& operator=(OpenedFileHandle&& other) noexcept {
+ std::swap(_cache, other._cache);
+ std::swap(_handle, other._handle);
+ return *this;
+ }
+
+ FileType* file() const {
+ DCHECK(_handle != nullptr);
+ return reinterpret_cast<FileType*>(_cache->value(_handle));
+ }
+
+private:
+ Cache* _cache;
+ Cache::Handle* _handle;
+};
+
+// Cache of open files.
+//
+// The purpose of this cache is to enforce an upper bound on the maximum number
+// of files open at a time. Files opened through the cache may be closed at any
+// time, only to be reopened upon next use.
+//
+// The file cache can be viewed as having two logical parts: the client-facing
+// File handle and the LRU cache.
+//
+// Client-facing API
+// -----------------
+// The core of the client-facing API is the cache descriptor. A descriptor
+
+// LRU cache
+// ---------
+// The lower half of the file cache is a standard LRU cache whose keys are file
+// names and whose values are pointers to opened file objects allocated on the
+// heap. Unlike the descriptor map, this cache has an upper bound on capacity,
+// and handles are evicted (and closed) according to an LRU algorithm.
+//
+// Whenever a descriptor is used by a client in file I/O, its file name is used
+// in an LRU cache lookup. If found, the underlying file is still open and the
+// file access is performed. Otherwise, the file must have been evicted and
+// closed, so it is reopened and reinserted (possibly evicting a different open
+// file) before the file access is performed.
+//
+// Every public method in the file cache is thread safe.
+template <class FileType>
+class FileCache {
+public:
+ // Creates a new file cache.
+ //
+ // The 'cache_name' is used to disambiguate amongst other file cache
+ // instances. The cache will use 'max_open_files' as a soft upper bound on
+ // the number of files open at any given time.
+ FileCache(const std::string& cache_name, int max_open_files);
+
+ // Destroys the file cache.
+ ~FileCache() { }
+
+ // find whether the file has been cached
+ // if cached, return true and set the file_handle
+ // else return false
+ bool lookup(const std::string& file_name, OpenedFileHandle<FileType>* file_handle);
+
+ // insert new FileType* into lru cache
+ // and return file_handle
+ void insert(const std::string& file_name, FileType* file, OpenedFileHandle<FileType>* file_handle);
+
+private:
+ // Name of the cache.
+ std::string _cache_name;
+
+ // Underlying cache instance. Caches opened files.
+ std::unique_ptr<Cache> _cache;
+
+ DISALLOW_COPY_AND_ASSIGN(FileCache);
+};
+
+} // namespace doris
diff --git a/be/src/util/file_manager.cpp b/be/src/util/file_manager.cpp
new file mode 100644
index 0000000..b9da240
--- /dev/null
+++ b/be/src/util/file_manager.cpp
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/file_manager.h"
+
+namespace doris {
+
+FileManager* FileManager::instance() {
+ static FileManager file_manager(Env::Default());
+ return &file_manager;
+}
+
+Status FileManager::open_file(const std::string& file_name, OpenedFileHandle<RandomAccessFile>* file_handle) {
+ bool found = _file_cache->lookup(file_name, file_handle);
+ if (found) {
+ return Status::OK();
+ }
+ std::unique_ptr<RandomAccessFile> file;
+ RETURN_IF_ERROR(_env->new_random_access_file(file_name, &file));
+ _file_cache->insert(file_name, file.release(), file_handle);
+ return Status::OK();
+}
+
+} // namespace doris
diff --git a/be/src/util/file_manager.h b/be/src/util/file_manager.h
new file mode 100644
index 0000000..c1d50dc
--- /dev/null
+++ b/be/src/util/file_manager.h
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "util/file_cache.h"
+#include "env/env.h"
+#include "common/config.h"
+
+namespace doris {
+
+class Env;
+
+// FileManager to control file operation
+// Now, only support RandomAccessFile for read operation for easy.
+class FileManager {
+public:
+ FileManager(Env* env) : _env(env),
+ _file_cache(new FileCache<RandomAccessFile>("Readable file cache", config::file_descriptor_cache_capacity)) { }
+
+ static FileManager* instance();
+
+ // Destroys the file cache.
+ ~FileManager() { }
+
+ Status open_file(const std::string& file_name, OpenedFileHandle<RandomAccessFile>* file_handle);
+
+private:
+ Env* _env;
+
+ // Underlying cache instance. Caches opened files.
+ std::unique_ptr<FileCache<RandomAccessFile>> _file_cache;
+
+ DISALLOW_COPY_AND_ASSIGN(FileManager);
+};
+
+} // namespace doris
diff --git a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
index 0174cfc..06371d3 100644
--- a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
@@ -81,11 +81,9 @@ void get_bloom_filter_reader_iter(const std::string& file_name, const BloomFilte
BloomFilterIndexReader** reader,
std::unique_ptr<BloomFilterIndexIterator>* iter) {
std::string fname = dname + "/" + file_name;
- auto st = Env::Default()->new_random_access_file(fname, rfile);
- ASSERT_TRUE(st.ok());
- *reader = new BloomFilterIndexReader(rfile->get(), bloom_filter_index_meta);
- st = (*reader)->load();
+ *reader = new BloomFilterIndexReader(fname, bloom_filter_index_meta);
+ auto st = (*reader)->load();
ASSERT_TRUE(st.ok());
st = (*reader)->new_iterator(iter);
diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
index d3d9573..c44ea9b 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
@@ -102,21 +102,21 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
// read and check
{
// read and check
- std::unique_ptr<RandomAccessFile> rfile;
- auto st = Env::Default()->new_random_access_file(fname, &rfile);
- ASSERT_TRUE(st.ok());
-
ColumnReaderOptions reader_opts;
std::unique_ptr<ColumnReader> reader;
- st = ColumnReader::create(reader_opts, meta, num_rows, rfile.get(), &reader);
+ auto st = ColumnReader::create(reader_opts, meta, num_rows, fname, &reader);
ASSERT_TRUE(st.ok());
ColumnIterator* iter = nullptr;
st = reader->new_iterator(&iter);
ASSERT_TRUE(st.ok());
+ std::unique_ptr<RandomAccessFile> rfile;
+ st = Env::Default()->new_random_access_file(fname, &rfile);
+ ASSERT_TRUE(st.ok());
ColumnIteratorOptions iter_opts;
OlapReaderStatistics stats;
iter_opts.stats = &stats;
+ iter_opts.file = rfile.get();
st = iter->init(iter_opts);
ASSERT_TRUE(st.ok());
// sequence read
diff --git a/be/test/olap/rowset/segment_v2/index_column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/index_column_reader_writer_test.cpp
index 4703cd3..c396462 100644
--- a/be/test/olap/rowset/segment_v2/index_column_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/index_column_reader_writer_test.cpp
@@ -68,15 +68,11 @@ void wirte_index_file(std::string& file_name, const void* values,
template<FieldType type>
void get_bitmap_reader_iter(std::string& file_name, BitmapIndexColumnPB& bitmap_index_meta,
- std::unique_ptr<RandomAccessFile>* rfile,
BitmapIndexReader** reader,
BitmapIndexIterator** iter) {
file_name = dname + "/" + file_name;
- auto st = Env::Default()->new_random_access_file(file_name, rfile);
- ASSERT_TRUE(st.ok());
-
- *reader = new BitmapIndexReader(rfile->get(), bitmap_index_meta);
- st = (*reader)->load();
+ *reader = new BitmapIndexReader(file_name, bitmap_index_meta);
+ auto st = (*reader)->load();
ASSERT_TRUE(st.ok());
st = (*reader)->new_iterator(iter);
@@ -98,8 +94,7 @@ TEST_F(IndexColumnReaderWriterTest, test_invert) {
std::unique_ptr<RandomAccessFile> rfile;
BitmapIndexReader* reader = nullptr;
BitmapIndexIterator* iter = nullptr;
- get_bitmap_reader_iter<OLAP_FIELD_TYPE_INT>(file_name, bitmap_index_meta,
- &rfile, &reader, &iter);
+ get_bitmap_reader_iter<OLAP_FIELD_TYPE_INT>(file_name, bitmap_index_meta, &reader, &iter);
int value = 2;
bool exact_match;
@@ -151,11 +146,9 @@ TEST_F(IndexColumnReaderWriterTest, test_invert_2) {
&bitmap_index_meta);
{
- std::unique_ptr<RandomAccessFile> rfile;
BitmapIndexReader* reader = nullptr;
BitmapIndexIterator* iter = nullptr;
- get_bitmap_reader_iter<OLAP_FIELD_TYPE_INT>(file_name, bitmap_index_meta,
- &rfile, &reader, &iter);
+ get_bitmap_reader_iter<OLAP_FIELD_TYPE_INT>(file_name, bitmap_index_meta, &reader, &iter);
int value = 1026;
bool exact_match;
@@ -187,11 +180,9 @@ TEST_F(IndexColumnReaderWriterTest, test_multi_pages) {
wirte_index_file<OLAP_FIELD_TYPE_BIGINT>(file_name, val, num_uint8_rows, 0,
&bitmap_index_meta);
{
- std::unique_ptr<RandomAccessFile> rfile;
BitmapIndexReader* reader = nullptr;
BitmapIndexIterator* iter = nullptr;
- get_bitmap_reader_iter<OLAP_FIELD_TYPE_BIGINT>(file_name, bitmap_index_meta,
- &rfile, &reader, &iter);
+ get_bitmap_reader_iter<OLAP_FIELD_TYPE_BIGINT>(file_name, bitmap_index_meta, &reader, &iter);
int64_t value = 2019;
bool exact_match;
@@ -220,11 +211,9 @@ TEST_F(IndexColumnReaderWriterTest, test_null) {
wirte_index_file<OLAP_FIELD_TYPE_BIGINT>(file_name, val, num_uint8_rows, 30,
&bitmap_index_meta);
{
- std::unique_ptr<RandomAccessFile> rfile;
BitmapIndexReader* reader = nullptr;
BitmapIndexIterator* iter = nullptr;
- get_bitmap_reader_iter<OLAP_FIELD_TYPE_BIGINT>(file_name, bitmap_index_meta,
- &rfile, &reader, &iter);
+ get_bitmap_reader_iter<OLAP_FIELD_TYPE_BIGINT>(file_name, bitmap_index_meta, &reader, &iter);
Roaring bitmap;
iter->read_null_bitmap(&bitmap);
diff --git a/be/test/util/CMakeLists.txt b/be/test/util/CMakeLists.txt
index 3154de5..db624a1 100644
--- a/be/test/util/CMakeLists.txt
+++ b/be/test/util/CMakeLists.txt
@@ -57,4 +57,5 @@ ADD_BE_TEST(zip_util_test)
ADD_BE_TEST(utf8_check_test)
ADD_BE_TEST(cgroup_util_test)
ADD_BE_TEST(path_util_test)
-
+ADD_BE_TEST(file_cache_test)
+ADD_BE_TEST(file_manager_test)
diff --git a/be/test/util/file_cache_test.cpp b/be/test/util/file_cache_test.cpp
new file mode 100644
index 0000000..efb4aa6
--- /dev/null
+++ b/be/test/util/file_cache_test.cpp
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/file_cache.h"
+
+#include <gtest/gtest.h>
+
+#include "env/env.h"
+
+namespace doris {
+
+class FileCacheTest : public testing::Test {
+public:
+ FileCacheTest() { }
+
+ void SetUp() override {
+ _file_cache.reset(new FileCache<RandomAccessFile>("test cache", 10000));
+ _file_exist = "file_exist";
+ std::unique_ptr<WritableFile> file;
+ auto st = Env::Default()->new_writable_file(_file_exist, &file);
+ ASSERT_TRUE(st.ok());
+ st = file->close();
+ ASSERT_TRUE(st.ok());
+ }
+
+ void TearDown() override {
+ _file_cache.reset(nullptr);
+ auto st = Env::Default()->delete_file(_file_exist);
+ ASSERT_TRUE(st.ok());
+ }
+
+private:
+ std::unique_ptr<FileCache<RandomAccessFile>> _file_cache;
+ std::string _file_exist;
+};
+
+TEST_F(FileCacheTest, normal) {
+ OpenedFileHandle<RandomAccessFile> file_handle;
+ auto found = _file_cache->lookup(_file_exist, &file_handle);
+ ASSERT_FALSE(found);
+ std::unique_ptr<RandomAccessFile> file;
+ auto st = Env::Default()->new_random_access_file(_file_exist, &file);
+ ASSERT_TRUE(st.ok());
+ RandomAccessFile* tmp_file = file.release();
+ _file_cache->insert(_file_exist, tmp_file, &file_handle);
+ ASSERT_EQ(tmp_file, file_handle.file());
+ OpenedFileHandle<RandomAccessFile> file_handle2;
+ found = _file_cache->lookup(_file_exist, &file_handle2);
+ ASSERT_EQ(file_handle.file(), file_handle2.file());
+}
+
+}
+
+int main(int argc, char* argv[]) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/be/test/util/file_manager_test.cpp b/be/test/util/file_manager_test.cpp
new file mode 100644
index 0000000..06ff580
--- /dev/null
+++ b/be/test/util/file_manager_test.cpp
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/file_manager.h"
+
+#include <gtest/gtest.h>
+
+#include "env/env.h"
+
+namespace doris {
+
+class FileManagerTest : public testing::Test {
+public:
+ FileManagerTest() { }
+
+ void SetUp() override {
+ _file_exist = "file_exist";
+ }
+
+ void TearDown() override {
+
+ }
+
+private:
+ std::string _file_exist;
+};
+
+TEST_F(FileManagerTest, normal) {
+ FileManager* file_manager = FileManager::instance();
+ OpenedFileHandle<RandomAccessFile> rfile;
+ auto st = file_manager->open_file(_file_exist, &rfile);
+ ASSERT_FALSE(st.ok());
+
+ std::unique_ptr<WritableFile> wfile;
+ st = Env::Default()->new_writable_file(_file_exist, &wfile);
+ ASSERT_TRUE(st.ok());
+ st = wfile->close();
+ ASSERT_TRUE(st.ok());
+
+ st = file_manager->open_file(_file_exist, &rfile);
+ ASSERT_TRUE(st.ok());
+ ASSERT_NE(nullptr, rfile.file());
+
+ st = Env::Default()->delete_file(_file_exist);
+ ASSERT_TRUE(st.ok());
+}
+
+}
+
+int main(int argc, char* argv[]) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/run-ut.sh b/run-ut.sh
index 297283d..c621aa1 100755
--- a/run-ut.sh
+++ b/run-ut.sh
@@ -168,6 +168,8 @@ ${DORIS_TEST_BINARY_DIR}/util/zip_util_test
${DORIS_TEST_BINARY_DIR}/util/utf8_check_test
${DORIS_TEST_BINARY_DIR}/util/cgroup_util_test
${DORIS_TEST_BINARY_DIR}/util/path_util_test
+${DORIS_TEST_BINARY_DIR}/util/file_cache_test
+${DORIS_TEST_BINARY_DIR}/util/file_manager_test
# Running common Unittest
${DORIS_TEST_BINARY_DIR}/common/resource_tls_test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org