You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/03/21 13:08:47 UTC
[doris] branch master updated: [refactor](file-system)(step-1) refactor file sysmte on BE and remove storage_backend (#17586)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cb79e42e5c [refactor](file-system)(step-1) refactor file sysmte on BE and remove storage_backend (#17586)
cb79e42e5c is described below
commit cb79e42e5cfe738531a8bca10ec8cf91e06d0ad5
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Tue Mar 21 21:08:38 2023 +0800
[refactor](file-system)(step-1) refactor file sysmte on BE and remove storage_backend (#17586)
See #17764 for details
I have tested:
- Unit test for local/s3/hdfs/broker file system: be/test/io/fs/file_system_test.cpp
- Outfile to local/s3/hdfs/broker.
- Load from local/s3/hdfs/broker.
- Query file on local/s3/hdfs/broker file system, with table value function and catalog.
- Backup/Restore with local/s3/hdfs/broker file system
Not test:
- cold & host data separation case.
---
be/src/agent/task_worker_pool.cpp | 32 +-
be/src/common/status.cpp | 8 +-
be/src/common/status.h | 6 +-
be/src/exec/arrow/arrow_reader.cpp | 4 +-
be/src/exec/line_reader.h | 7 +-
be/src/http/web_page_handler.cpp | 1 -
be/src/io/CMakeLists.txt | 17 +-
be/src/io/cache/block/block_file_segment.cpp | 7 +-
be/src/io/cache/block/block_lru_file_cache.cpp | 5 +-
.../io/cache/block/cached_remote_file_reader.cpp | 37 +-
be/src/io/cache/block/cached_remote_file_reader.h | 14 +-
be/src/io/cache/dummy_file_cache.h | 11 +-
be/src/io/cache/file_cache.cpp | 25 +-
be/src/io/cache/file_cache.h | 5 +
be/src/io/cache/file_cache_manager.cpp | 11 +-
be/src/io/cache/sub_file_cache.cpp | 23 +-
be/src/io/cache/sub_file_cache.h | 9 +-
be/src/io/cache/whole_file_cache.cpp | 25 +-
be/src/io/cache/whole_file_cache.h | 9 +-
be/src/io/file_factory.cpp | 93 ++-
be/src/io/file_factory.h | 20 +-
be/src/io/fs/broker_file_reader.cpp | 4 +-
be/src/io/fs/broker_file_reader.h | 7 +-
be/src/io/fs/broker_file_system.cpp | 311 +++++++---
be/src/io/fs/broker_file_system.h | 65 +--
.../broker_file_writer.cpp} | 203 ++++---
.../{broker_writer.h => fs/broker_file_writer.h} | 37 +-
be/src/io/{ => fs}/buffered_reader.cpp | 7 +-
be/src/io/{ => fs}/buffered_reader.h | 2 +
.../{file_reader_options.cpp => file_reader.cpp} | 23 +-
be/src/io/fs/file_reader.h | 18 +-
be/src/io/fs/file_reader_options.cpp | 3 +
be/src/io/fs/file_reader_options.h | 2 +
.../fs/file_reader_writer_fwd.h} | 19 +-
be/src/io/fs/file_system.cpp | 154 +++++
be/src/io/fs/file_system.h | 128 ++--
be/src/io/fs/file_writer.h | 19 +-
.../fs/{file_reader_options.cpp => fs_utils.cpp} | 32 +-
be/src/io/{file_writer.h => fs/fs_utils.h} | 24 +-
be/src/io/fs/hdfs_file_reader.cpp | 4 +-
be/src/io/fs/hdfs_file_reader.h | 8 +-
be/src/io/fs/hdfs_file_system.cpp | 289 ++++++---
be/src/io/fs/hdfs_file_system.h | 64 +-
be/src/io/fs/hdfs_file_writer.cpp | 139 +++++
be/src/io/{hdfs_writer.h => fs/hdfs_file_writer.h} | 34 +-
be/src/io/fs/local_file_reader.cpp | 14 +-
be/src/io/fs/local_file_reader.h | 9 +-
be/src/io/fs/local_file_system.cpp | 224 +++----
be/src/io/fs/local_file_system.h | 61 +-
be/src/io/fs/local_file_writer.cpp | 66 +--
be/src/io/fs/local_file_writer.h | 17 +-
be/src/io/fs/remote_file_system.cpp | 88 ++-
be/src/io/fs/remote_file_system.h | 55 +-
be/src/io/fs/s3_file_reader.cpp | 15 +-
be/src/io/fs/s3_file_reader.h | 9 +-
be/src/io/fs/s3_file_system.cpp | 548 ++++++++++--------
be/src/io/fs/s3_file_system.h | 99 ++--
be/src/io/fs/s3_file_writer.cpp | 80 ++-
be/src/io/fs/s3_file_writer.h | 24 +-
be/src/io/fs/stream_load_pipe.cpp | 7 +-
be/src/io/fs/stream_load_pipe.h | 7 +-
be/src/io/hdfs_writer.cpp | 154 -----
be/src/io/io_common.h | 69 +++
be/src/io/local_file_writer.cpp | 88 ---
be/src/io/local_file_writer.h | 48 --
be/src/io/s3_writer.cpp | 138 -----
be/src/io/s3_writer.h | 59 --
be/src/olap/CMakeLists.txt | 1 -
be/src/olap/data_dir.cpp | 1 -
be/src/olap/file_header.h | 240 ++++++++
be/src/olap/file_helper.cpp | 377 ------------
be/src/olap/file_helper.h | 430 --------------
be/src/olap/iterators.h | 22 +-
be/src/olap/merger.h | 1 +
be/src/olap/olap_common.h | 24 +-
be/src/olap/push_handler.h | 1 -
be/src/olap/reader.h | 1 +
be/src/olap/rowset/beta_rowset.cpp | 8 +-
be/src/olap/rowset/rowset_reader_context.h | 1 +
be/src/olap/rowset/segment_v2/column_reader.h | 3 +-
.../inverted_index_compound_directory.cpp | 15 +-
be/src/olap/rowset/segment_v2/page_io.cpp | 4 +-
be/src/olap/rowset/segment_v2/page_io.h | 2 +-
be/src/olap/rowset/segment_v2/segment.cpp | 15 +-
be/src/olap/storage_policy.h | 2 +-
be/src/olap/tablet.cpp | 29 +-
be/src/olap/tablet_manager.cpp | 2 +-
be/src/olap/tablet_meta.cpp | 43 +-
be/src/olap/utils.cpp | 44 +-
.../routine_load/routine_load_task_executor.cpp | 3 +-
be/src/runtime/snapshot_loader.cpp | 73 ++-
be/src/runtime/snapshot_loader.h | 16 +-
be/src/service/internal_service.cpp | 4 +-
be/src/util/CMakeLists.txt | 4 -
be/src/util/async_io.h | 8 +-
be/src/util/broker_storage_backend.cpp | 417 -------------
be/src/util/broker_storage_backend.h | 60 --
be/src/util/file_utils.cpp | 1 -
be/src/util/hdfs_storage_backend.cpp | 335 -----------
be/src/util/hdfs_storage_backend.h | 62 --
be/src/util/hdfs_util.cpp | 11 +
be/src/util/hdfs_util.h | 9 +-
be/src/util/s3_storage_backend.cpp | 307 ----------
be/src/util/s3_storage_backend.h | 60 --
be/src/util/s3_uri.cpp | 27 +-
be/src/util/s3_uri.h | 3 +-
be/src/util/s3_util.cpp | 20 +-
be/src/util/s3_util.h | 8 +-
be/src/util/storage_backend.h | 53 --
be/src/util/storage_backend_mgr.cpp | 217 -------
be/src/util/storage_backend_mgr.h | 77 ---
be/src/vec/core/block_spill_reader.cpp | 11 +-
be/src/vec/core/block_spill_writer.cpp | 13 +-
be/src/vec/core/block_spill_writer.h | 6 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 17 +-
be/src/vec/exec/format/csv/csv_reader.h | 6 +-
.../file_reader/new_plain_binary_line_reader.cpp | 3 +-
.../file_reader/new_plain_binary_line_reader.h | 3 +-
.../file_reader/new_plain_text_line_reader.cpp | 6 +-
.../file_reader/new_plain_text_line_reader.h | 6 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 22 +-
be/src/vec/exec/format/json/new_json_reader.h | 6 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 14 +-
be/src/vec/exec/format/orc/vorc_reader.h | 14 +-
.../vec/exec/format/parquet/parquet_thrift_util.h | 8 +-
.../parquet/vparquet_column_chunk_reader.cpp | 2 +-
.../format/parquet/vparquet_column_chunk_reader.h | 6 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 4 +-
.../exec/format/parquet/vparquet_column_reader.h | 6 +-
.../exec/format/parquet/vparquet_page_reader.cpp | 2 +-
.../vec/exec/format/parquet/vparquet_page_reader.h | 6 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 14 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 6 +-
be/src/vec/exec/format/table/iceberg_reader.cpp | 2 +-
be/src/vec/exec/format/table/iceberg_reader.h | 7 +-
be/src/vec/exec/scan/vfile_scanner.cpp | 4 +-
be/src/vec/exec/scan/vfile_scanner.h | 4 +-
be/src/vec/exec/varrow_scanner.cpp | 6 +-
be/src/vec/olap/vertical_merge_iterator.h | 1 +
be/src/vec/runtime/vfile_result_writer.cpp | 29 +-
be/src/vec/runtime/vfile_result_writer.h | 7 +-
be/src/vec/runtime/vorc_writer.cpp | 13 +-
be/src/vec/runtime/vorc_writer.h | 13 +-
be/src/vec/runtime/vparquet_writer.cpp | 10 +-
be/src/vec/runtime/vparquet_writer.h | 10 +-
be/test/CMakeLists.txt | 6 +-
be/test/io/cache/remote_file_cache_test.cpp | 8 +-
be/test/io/fs/file_system_test.cpp | 643 +++++++++++++++++++++
be/test/io/fs/test_data/download_file.txt | 1 +
be/test/io/fs/test_data/upload_file.txt | 1 +
.../{file_helper_test.cpp => file_header_test.cpp} | 43 +-
be/test/olap/file_utils_test.cpp | 26 +-
be/test/olap/primary_key_index_test.cpp | 2 +-
be/test/olap/remote_rowset_gc_test.cpp | 6 +-
be/test/olap/rowset/beta_rowset_test.cpp | 7 +-
.../olap/rowset/segment_v2/bitmap_index_test.cpp | 3 +-
.../bloom_filter_index_reader_writer_test.cpp | 2 +-
.../segment_v2/column_reader_writer_test.cpp | 4 +-
.../rowset/segment_v2/ordinal_page_index_test.cpp | 2 +-
.../olap/rowset/segment_v2/zone_map_index_test.cpp | 6 +-
be/test/olap/tablet_cooldown_test.cpp | 98 ++--
be/test/util/broker_storage_backend_test.cpp | 190 ------
be/test/util/hdfs_storage_backend_test.cpp | 172 ------
be/test/util/s3_storage_backend_test.cpp | 190 ------
be/test/vec/exec/parquet/parquet_reader_test.cpp | 3 +-
be/test/vec/exec/parquet/parquet_thrift_test.cpp | 22 +-
.../java/org/apache/doris/backup/BackupJob.java | 4 +-
.../java/org/apache/doris/backup/RestoreJob.java | 2 +-
.../java/org/apache/doris/task/DownloadTask.java | 5 +-
.../java/org/apache/doris/task/UploadTask.java | 5 +-
.../org/apache/doris/backup/BackupHandlerTest.java | 4 +-
gensrc/thrift/AgentService.thrift | 3 +-
172 files changed, 3380 insertions(+), 5250 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 53a09999cd..78001d53c3 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1404,10 +1404,14 @@ void TaskWorkerPool::_upload_worker_thread_callback() {
std::map<int64_t, std::vector<std::string>> tablet_files;
std::unique_ptr<SnapshotLoader> loader = std::make_unique<SnapshotLoader>(
_env, upload_request.job_id, agent_task_req.signature, upload_request.broker_addr,
- upload_request.broker_prop,
+ upload_request.broker_prop);
+ Status status = loader->init(
upload_request.__isset.storage_backend ? upload_request.storage_backend
- : TStorageBackendType::type::BROKER);
- Status status = loader->upload(upload_request.src_dest_map, &tablet_files);
+ : TStorageBackendType::type::BROKER,
+ upload_request.__isset.location ? upload_request.location : "");
+ if (status.ok()) {
+ status = loader->upload(upload_request.src_dest_map, &tablet_files);
+ }
if (!status.ok()) {
LOG_WARNING("failed to upload")
@@ -1456,10 +1460,14 @@ void TaskWorkerPool::_download_worker_thread_callback() {
std::unique_ptr<SnapshotLoader> loader = std::make_unique<SnapshotLoader>(
_env, download_request.job_id, agent_task_req.signature,
- download_request.broker_addr, download_request.broker_prop,
+ download_request.broker_addr, download_request.broker_prop);
+ Status status = loader->init(
download_request.__isset.storage_backend ? download_request.storage_backend
- : TStorageBackendType::type::BROKER);
- Status status = loader->download(download_request.src_dest_map, &downloaded_tablet_ids);
+ : TStorageBackendType::type::BROKER,
+ download_request.__isset.location ? download_request.location : "");
+ if (status.ok()) {
+ status = loader->download(download_request.src_dest_map, &downloaded_tablet_ids);
+ }
if (!status.ok()) {
LOG_WARNING("failed to download")
@@ -1769,6 +1777,7 @@ void TaskWorkerPool::_push_storage_policy_worker_thread_callback() {
continue;
}
if (resource.__isset.s3_storage_param) {
+ Status st;
S3Conf s3_conf;
s3_conf.ak = std::move(resource.s3_storage_param.ak);
s3_conf.sk = std::move(resource.s3_storage_param.sk);
@@ -1779,14 +1788,13 @@ void TaskWorkerPool::_push_storage_policy_worker_thread_callback() {
s3_conf.connect_timeout_ms = resource.s3_storage_param.conn_timeout_ms;
s3_conf.max_connections = resource.s3_storage_param.max_conn;
s3_conf.request_timeout_ms = resource.s3_storage_param.request_timeout_ms;
- std::shared_ptr<io::S3FileSystem> s3_fs;
+ std::shared_ptr<io::S3FileSystem> fs;
if (existed_resource.fs == nullptr) {
- s3_fs = io::S3FileSystem::create(s3_conf, std::to_string(resource.id));
+ st = io::S3FileSystem::create(s3_conf, std::to_string(resource.id), &fs);
} else {
- s3_fs = std::static_pointer_cast<io::S3FileSystem>(existed_resource.fs);
- s3_fs->set_conf(s3_conf);
+ fs = std::static_pointer_cast<io::S3FileSystem>(existed_resource.fs);
+ fs->set_conf(s3_conf);
}
- auto st = s3_fs->connect();
if (!st.ok()) {
LOG(WARNING) << "update s3 resource failed: " << st;
} else {
@@ -1794,7 +1802,7 @@ void TaskWorkerPool::_push_storage_policy_worker_thread_callback() {
.tag("resource_id", resource.id)
.tag("resource_name", resource.name)
.tag("s3_conf", s3_conf.to_string());
- put_storage_resource(resource.id, {std::move(s3_fs), resource.version});
+ put_storage_resource(resource.id, {std::move(fs), resource.version});
}
} else {
LOG(WARNING) << "unknown resource=" << resource;
diff --git a/be/src/common/status.cpp b/be/src/common/status.cpp
index 32aafe35ed..85f4d69938 100644
--- a/be/src/common/status.cpp
+++ b/be/src/common/status.cpp
@@ -8,6 +8,7 @@
#include <rapidjson/stringbuffer.h>
#include "gen_cpp/types.pb.h" // for PStatus
+#include "service/backend_options.h"
namespace doris {
@@ -52,8 +53,8 @@ void Status::to_thrift(TStatus* s) const {
return;
}
s->status_code = (int16_t)_code > 0 ? (TStatusCode::type)_code : TStatusCode::INTERNAL_ERROR;
- s->error_msgs.push_back(
- fmt::format("[{}]{}", code_as_string(), _err_msg ? _err_msg->_msg : ""));
+ s->error_msgs.push_back(fmt::format("({})[{}]{}", BackendOptions::get_localhost(),
+ code_as_string(), _err_msg ? _err_msg->_msg : ""));
s->__isset.error_msgs = true;
}
@@ -67,7 +68,8 @@ void Status::to_protobuf(PStatus* s) const {
s->clear_error_msgs();
s->set_status_code((int)_code);
if (!ok() && _err_msg) {
- s->add_error_msgs(_err_msg->_msg);
+ s->add_error_msgs(fmt::format("({})[{}]{}", BackendOptions::get_localhost(),
+ code_as_string(), _err_msg ? _err_msg->_msg : ""));
}
}
diff --git a/be/src/common/status.h b/be/src/common/status.h
index 772eaa2230..12aa7b6b6c 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -13,7 +13,6 @@
#include "common/compiler_util.h"
#include "gen_cpp/Status_types.h" // for TStatus
-#include "service/backend_options.h"
#ifdef ENABLE_STACKTRACE
#include "util/stack_util.h"
#endif
@@ -301,7 +300,6 @@ public:
if (rhs._err_msg) {
_err_msg = std::make_unique<ErrMsg>(*rhs._err_msg);
}
- _be_ip = rhs._be_ip;
return *this;
}
@@ -479,12 +477,10 @@ private:
#endif
};
std::unique_ptr<ErrMsg> _err_msg;
- std::string_view _be_ip = BackendOptions::get_localhost();
};
inline std::ostream& operator<<(std::ostream& ostr, const Status& status) {
ostr << '[' << status.code_as_string() << ']';
- ostr << '[' << status._be_ip << ']';
ostr << (status._err_msg ? status._err_msg->_msg : "");
#ifdef ENABLE_STACKTRACE
if (status._err_msg && !status._err_msg->_stack.empty()) {
@@ -510,7 +506,7 @@ inline std::string Status::to_string() const {
} while (false)
#define RETURN_ERROR_IF_NON_VEC \
- return Status::NotSupported("Non-vectorized engine is not supported since Doris 1.3+.");
+ return Status::NotSupported("Non-vectorized engine is not supported since Doris 2.0.");
// End _get_next_span after last call to get_next method
#define RETURN_IF_ERROR_AND_CHECK_SPAN(stmt, get_next_span, done) \
diff --git a/be/src/exec/arrow/arrow_reader.cpp b/be/src/exec/arrow/arrow_reader.cpp
index c03d7791dd..d0dbf3f992 100644
--- a/be/src/exec/arrow/arrow_reader.cpp
+++ b/be/src/exec/arrow/arrow_reader.cpp
@@ -23,6 +23,7 @@
#include "common/logging.h"
#include "gen_cpp/PaloBrokerService_types.h"
#include "gen_cpp/TPaloBrokerService.h"
+#include "io/io_common.h"
#include "olap/iterators.h"
#include "runtime/broker_mgr.h"
#include "runtime/client_cache.h"
@@ -216,8 +217,7 @@ arrow::Result<int64_t> ArrowFile::ReadAt(int64_t position, int64_t nbytes, void*
while (bytes_read < nbytes) {
size_t reads = 0;
Slice file_slice((uint8_t*)out, nbytes);
- IOContext io_ctx;
- Status result = _file_reader->read_at(_pos, file_slice, io_ctx, &reads);
+ Status result = _file_reader->read_at(_pos, file_slice, &reads);
if (!result.ok()) {
return arrow::Status::IOError("Readat failed.");
}
diff --git a/be/src/exec/line_reader.h b/be/src/exec/line_reader.h
index 80a8f94977..0efdf43e57 100644
--- a/be/src/exec/line_reader.h
+++ b/be/src/exec/line_reader.h
@@ -20,12 +20,15 @@
#include "common/status.h"
namespace doris {
-
+namespace io {
+class IOContext;
+}
// This class is used for CSV scanner, to read content line by line
class LineReader {
public:
virtual ~LineReader() = default;
- virtual Status read_line(const uint8_t** ptr, size_t* size, bool* eof) = 0;
+ virtual Status read_line(const uint8_t** ptr, size_t* size, bool* eof,
+ const io::IOContext* io_ctx) = 0;
virtual void close() = 0;
};
diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp
index 9c81f2a1c7..4a5b873a08 100644
--- a/be/src/http/web_page_handler.cpp
+++ b/be/src/http/web_page_handler.cpp
@@ -30,7 +30,6 @@
#include "http/http_response.h"
#include "http/http_status.h"
#include "http/utils.h"
-#include "olap/file_helper.h"
#include "util/cpu_info.h"
#include "util/debug_util.h"
#include "util/disk_info.h"
diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt
index 3dae0d9263..1c953f1e55 100644
--- a/be/src/io/CMakeLists.txt
+++ b/be/src/io/CMakeLists.txt
@@ -22,26 +22,27 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io")
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io")
set(IO_FILES
- broker_writer.cpp
- buffered_reader.cpp
file_factory.cpp
hdfs_builder.cpp
- hdfs_writer.cpp
- local_file_writer.cpp
- s3_writer.cpp
fs/file_reader_options.cpp
- fs/local_file_reader.cpp
+ fs/file_reader.cpp
+ fs/file_system.cpp
+ fs/remote_file_system.cpp
fs/local_file_system.cpp
+ fs/local_file_reader.cpp
fs/local_file_writer.cpp
- fs/s3_file_reader.cpp
fs/s3_file_system.cpp
+ fs/s3_file_reader.cpp
fs/s3_file_writer.cpp
fs/hdfs_file_system.cpp
fs/hdfs_file_reader.cpp
+ fs/hdfs_file_writer.cpp
fs/broker_file_system.cpp
fs/broker_file_reader.cpp
- fs/remote_file_system.cpp
+ fs/broker_file_writer.cpp
+ fs/buffered_reader.cpp
fs/stream_load_pipe.cpp
+ fs/fs_utils.cpp
cache/dummy_file_cache.cpp
cache/file_cache.cpp
cache/file_cache_manager.cpp
diff --git a/be/src/io/cache/block/block_file_segment.cpp b/be/src/io/cache/block/block_file_segment.cpp
index ae98270897..020fc85de1 100644
--- a/be/src/io/cache/block/block_file_segment.cpp
+++ b/be/src/io/cache/block/block_file_segment.cpp
@@ -26,7 +26,6 @@
#include <thread>
#include "common/status.h"
-#include "io/fs/file_reader.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "olap/iterators.h"
@@ -171,13 +170,11 @@ Status FileBlock::read_at(Slice buffer, size_t offset) {
std::lock_guard segment_lock(_mutex);
if (!_cache_reader) {
auto download_path = get_path_in_local_cache();
- RETURN_IF_ERROR(
- global_local_filesystem()->open_file(download_path, &_cache_reader, nullptr));
+ RETURN_IF_ERROR(global_local_filesystem()->open_file(download_path, &_cache_reader));
}
}
size_t bytes_reads = buffer.size;
- IOContext io_ctx;
- RETURN_IF_ERROR(_cache_reader->read_at(offset, buffer, io_ctx, &bytes_reads));
+ RETURN_IF_ERROR(_cache_reader->read_at(offset, buffer, &bytes_reads));
DCHECK(bytes_reads == buffer.size);
return Status::OK();
}
diff --git a/be/src/io/cache/block/block_lru_file_cache.cpp b/be/src/io/cache/block/block_lru_file_cache.cpp
index e2b1063007..22dbfe80f0 100644
--- a/be/src/io/cache/block/block_lru_file_cache.cpp
+++ b/be/src/io/cache/block/block_lru_file_cache.cpp
@@ -790,9 +790,8 @@ std::string LRUFileCache::read_file_cache_version() const {
fs->file_size(version_path, &file_size);
char version[file_size];
- IOContext io_ctx;
- fs->open_file(version_path, &version_reader, &io_ctx);
- version_reader->read_at(0, Slice(version, file_size), io_ctx, &file_size);
+ fs->open_file(version_path, &version_reader);
+ version_reader->read_at(0, Slice(version, file_size), &file_size);
version_reader->close();
return std::string(version, file_size);
}
diff --git a/be/src/io/cache/block/cached_remote_file_reader.cpp b/be/src/io/cache/block/cached_remote_file_reader.cpp
index 1b82916e11..c3a3eee9f6 100644
--- a/be/src/io/cache/block/cached_remote_file_reader.cpp
+++ b/be/src/io/cache/block/cached_remote_file_reader.cpp
@@ -28,8 +28,8 @@ namespace doris {
namespace io {
CachedRemoteFileReader::CachedRemoteFileReader(FileReaderSPtr remote_file_reader,
- const std::string& cache_path, IOContext* io_ctx)
- : _remote_file_reader(std::move(remote_file_reader)), _io_ctx(io_ctx) {
+ const std::string& cache_path)
+ : _remote_file_reader(std::move(remote_file_reader)) {
_cache_key = IFileCache::hash(cache_path);
_cache = FileCacheFactory::instance().get_by_path(_cache_key);
_disposable_cache = FileCacheFactory::instance().get_disposable_cache(_cache_key);
@@ -57,21 +57,10 @@ std::pair<size_t, size_t> CachedRemoteFileReader::_align_size(size_t offset,
return std::make_pair(align_left, align_size);
}
-Status CachedRemoteFileReader::read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
- if (bthread_self() == 0) {
- return read_at_impl(offset, result, io_ctx, bytes_read);
- }
- Status s;
- auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result,
- const IOContext& /*io_ctx*/, size_t* bytes_read) {
+Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) {
DCHECK(!closed());
- DCHECK(_io_ctx);
+ DCHECK(io_ctx);
if (offset > size()) {
return Status::IOError(
fmt::format("offset exceeds file size(offset: {), file size: {}, path: {})", offset,
@@ -83,23 +72,23 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result,
*bytes_read = 0;
return Status::OK();
}
- CloudFileCachePtr cache = _io_ctx->use_disposable_cache ? _disposable_cache : _cache;
+ CloudFileCachePtr cache = io_ctx->use_disposable_cache ? _disposable_cache : _cache;
// cache == nullptr since use_disposable_cache = true and don't set disposable cache in conf
if (cache == nullptr) {
- return _remote_file_reader->read_at(offset, result, *_io_ctx, bytes_read);
+ return _remote_file_reader->read_at(offset, result, bytes_read, io_ctx);
}
ReadStatistics stats;
stats.bytes_read = bytes_req;
// if state == nullptr, the method is called for read footer
// if state->read_segment_index, read all the end of file
size_t align_left = offset, align_size = size() - offset;
- if (!_io_ctx->read_segment_index) {
+ if (!io_ctx->read_segment_index) {
auto pair = _align_size(offset, bytes_req);
align_left = pair.first;
align_size = pair.second;
}
- bool is_persistent = _io_ctx->is_persistent;
- TUniqueId query_id = _io_ctx->query_id ? *(_io_ctx->query_id) : TUniqueId();
+ bool is_persistent = io_ctx->is_persistent;
+ TUniqueId query_id = io_ctx->query_id ? *(io_ctx->query_id) : TUniqueId();
FileBlocksHolder holder =
cache->get_or_set(_cache_key, align_left, align_size, is_persistent, query_id);
std::vector<FileBlockSPtr> empty_segments;
@@ -122,8 +111,8 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result,
empty_end = empty_segments.back()->range().right;
size_t size = empty_end - empty_start + 1;
std::unique_ptr<char[]> buffer(new char[size]);
- RETURN_IF_ERROR(_remote_file_reader->read_at(empty_start, Slice(buffer.get(), size),
- *_io_ctx, &size));
+ RETURN_IF_ERROR(_remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size,
+ io_ctx));
for (auto& segment : empty_segments) {
if (segment->state() == FileBlock::State::SKIP_CACHE) {
continue;
@@ -194,7 +183,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result,
current_offset = right + 1;
}
DCHECK(*bytes_read == bytes_req);
- _update_state(stats, _io_ctx->file_cache_stats);
+ _update_state(stats, io_ctx->file_cache_stats);
DorisMetrics::instance()->s3_bytes_read_total->increment(*bytes_read);
return Status::OK();
}
diff --git a/be/src/io/cache/block/cached_remote_file_reader.h b/be/src/io/cache/block/cached_remote_file_reader.h
index 496f151404..179a61a066 100644
--- a/be/src/io/cache/block/cached_remote_file_reader.h
+++ b/be/src/io/cache/block/cached_remote_file_reader.h
@@ -31,18 +31,12 @@ namespace io {
class CachedRemoteFileReader final : public FileReader {
public:
- CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const std::string& cache_path,
- IOContext* io_ctx);
+ CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const std::string& cache_path);
~CachedRemoteFileReader() override;
Status close() override;
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
- Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read);
-
const Path& path() const override { return _remote_file_reader->path(); }
size_t size() const override { return _remote_file_reader->size(); }
@@ -51,6 +45,10 @@ public:
FileSystemSPtr fs() const override { return _remote_file_reader->fs(); }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
std::pair<size_t, size_t> _align_size(size_t offset, size_t size) const;
@@ -59,8 +57,6 @@ private:
CloudFileCachePtr _cache;
CloudFileCachePtr _disposable_cache;
- IOContext* _io_ctx;
-
struct ReadStatistics {
bool hit_cache = false;
int64_t bytes_read = 0;
diff --git a/be/src/io/cache/dummy_file_cache.h b/be/src/io/cache/dummy_file_cache.h
index 7af7451130..653d47b1d6 100644
--- a/be/src/io/cache/dummy_file_cache.h
+++ b/be/src/io/cache/dummy_file_cache.h
@@ -38,11 +38,6 @@ public:
Status close() override { return Status::OK(); }
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override {
- return Status::NotSupported("dummy file cache only used for GC");
- }
-
const Path& path() const override { return _cache_dir; }
size_t size() const override { return 0; }
@@ -71,6 +66,12 @@ public:
FileSystemSPtr fs() const override { return nullptr; }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override {
+ return Status::NotSupported("dummy file cache only used for GC");
+ }
+
private:
void _add_file_cache(const Path& data_file);
void _load();
diff --git a/be/src/io/cache/file_cache.cpp b/be/src/io/cache/file_cache.cpp
index 5a4382a6f7..4dc08b52c9 100644
--- a/be/src/io/cache/file_cache.cpp
+++ b/be/src/io/cache/file_cache.cpp
@@ -21,6 +21,7 @@
#include "common/status.h"
#include "gutil/strings/util.h"
#include "io/fs/local_file_system.h"
+#include "io/fs/local_file_writer.h"
#include "olap/iterators.h"
namespace doris {
@@ -50,10 +51,8 @@ Status FileCache::download_cache_to_local(const Path& cache_file, const Path& ca
}
Slice file_slice(file_buf, need_req_size);
size_t bytes_read = 0;
- IOContext io_ctx;
RETURN_NOT_OK_STATUS_WITH_WARN(
- remote_file_reader->read_at(offset + count_bytes_read, file_slice, io_ctx,
- &bytes_read),
+ remote_file_reader->read_at(offset + count_bytes_read, file_slice, &bytes_read),
fmt::format("read remote file failed. {}. offset: {}, request size: {}",
remote_file_reader->path().native(), offset + count_bytes_read,
need_req_size));
@@ -129,19 +128,20 @@ Status FileCache::_get_dir_files_and_remove_unfinished(const Path& cache_dir,
}
// list all files
- std::vector<Path> cache_file_names;
+ std::vector<FileInfo> cache_files;
+ bool exists = true;
RETURN_NOT_OK_STATUS_WITH_WARN(
- io::global_local_filesystem()->list(cache_dir, &cache_file_names),
+ io::global_local_filesystem()->list(cache_dir, true, &cache_files, &exists),
fmt::format("List dir failed: {}", cache_dir.native()))
// separate DATA file and DONE file
std::set<Path> cache_names_temp;
std::list<Path> done_names_temp;
- for (auto& cache_file_name : cache_file_names) {
- if (ends_with(cache_file_name.native(), CACHE_DONE_FILE_SUFFIX)) {
- done_names_temp.push_back(std::move(cache_file_name));
+ for (auto& cache_file : cache_files) {
+ if (ends_with(cache_file.file_name, CACHE_DONE_FILE_SUFFIX)) {
+ done_names_temp.push_back(cache_file.file_name);
} else {
- cache_names_temp.insert(std::move(cache_file_name));
+ cache_names_temp.insert(cache_file.file_name);
}
}
@@ -181,11 +181,12 @@ Status FileCache::_check_and_delete_empty_dir(const Path& cache_dir) {
return Status::OK();
}
- std::vector<Path> cache_file_names;
+ std::vector<FileInfo> cache_files;
+ bool exists = true;
RETURN_NOT_OK_STATUS_WITH_WARN(
- io::global_local_filesystem()->list(cache_dir, &cache_file_names),
+ io::global_local_filesystem()->list(cache_dir, true, &cache_files, &exists),
fmt::format("List dir failed: {}", cache_dir.native()));
- if (cache_file_names.empty()) {
+ if (cache_files.empty()) {
RETURN_NOT_OK_STATUS_WITH_WARN(io::global_local_filesystem()->delete_directory(cache_dir),
fmt::format("Delete dir failed: {}", cache_dir.native()));
LOG(INFO) << "Delete empty dir: " << cache_dir.native();
diff --git a/be/src/io/cache/file_cache.h b/be/src/io/cache/file_cache.h
index 692cd00e16..68ffc1a3a1 100644
--- a/be/src/io/cache/file_cache.h
+++ b/be/src/io/cache/file_cache.h
@@ -61,6 +61,11 @@ public:
virtual int64_t get_oldest_match_time() const = 0;
protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override {
+ return Status::NotSupported("dummy file cache only used for GC");
+ }
+
Status _remove_file(const Path& file, size_t* cleaned_size);
Status _remove_cache_and_done(const Path& cache_file, const Path& cache_done_file,
diff --git a/be/src/io/cache/file_cache_manager.cpp b/be/src/io/cache/file_cache_manager.cpp
index 33b6a2e145..d3bf4942de 100644
--- a/be/src/io/cache/file_cache_manager.cpp
+++ b/be/src/io/cache/file_cache_manager.cpp
@@ -120,11 +120,14 @@ void FileCacheManager::_add_file_cache_for_gc_by_disk(std::vector<GCContextPerDi
void FileCacheManager::_gc_unused_file_caches(std::list<FileCachePtr>& result) {
std::vector<TabletSharedPtr> tablets =
StorageEngine::instance()->tablet_manager()->get_all_tablet();
+ bool exists = true;
for (const auto& tablet : tablets) {
- std::vector<Path> seg_file_paths;
- if (io::global_local_filesystem()->list(tablet->tablet_path(), &seg_file_paths).ok()) {
- for (Path seg_file : seg_file_paths) {
- std::string seg_filename = seg_file.native();
+ std::vector<FileInfo> seg_files;
+ if (io::global_local_filesystem()
+ ->list(tablet->tablet_path(), true, &seg_files, &exists)
+ .ok()) {
+ for (auto& seg_file : seg_files) {
+ std::string seg_filename = seg_file.file_name;
// check if it is a dir name
if (!BetaRowset::is_segment_cache_dir(seg_filename)) {
continue;
diff --git a/be/src/io/cache/sub_file_cache.cpp b/be/src/io/cache/sub_file_cache.cpp
index 4b21e37505..f23e122132 100644
--- a/be/src/io/cache/sub_file_cache.cpp
+++ b/be/src/io/cache/sub_file_cache.cpp
@@ -49,22 +49,11 @@ SubFileCache::SubFileCache(const Path& cache_dir, int64_t alive_time_sec,
SubFileCache::~SubFileCache() {}
-Status SubFileCache::read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
- if (bthread_self() == 0) {
- return read_at_impl(offset, result, io_ctx, bytes_read);
- }
- Status s;
- auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status SubFileCache::read_at_impl(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
+Status SubFileCache::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) {
RETURN_IF_ERROR(_init());
- if (io_ctx.reader_type != READER_QUERY) {
- return _remote_file_reader->read_at(offset, result, io_ctx, bytes_read);
+ if (io_ctx->reader_type != READER_QUERY) {
+ return _remote_file_reader->read_at(offset, result, bytes_read, io_ctx);
}
std::vector<size_t> need_cache_offsets;
RETURN_IF_ERROR(_get_need_cache_offsets(offset, result.size, &need_cache_offsets));
@@ -125,7 +114,7 @@ Status SubFileCache::read_at_impl(size_t offset, Slice result, const IOContext&
Slice read_slice(result.mutable_data() + offset_begin - offset, req_size);
size_t sub_bytes_read = -1;
RETURN_NOT_OK_STATUS_WITH_WARN(
- _cache_file_readers[*iter]->read_at(offset_begin - *iter, read_slice, io_ctx,
+ _cache_file_readers[*iter]->read_at(offset_begin - *iter, read_slice,
&sub_bytes_read),
fmt::format("Read local cache file failed: {}",
_cache_file_readers[*iter]->path().native()));
@@ -204,7 +193,7 @@ Status SubFileCache::_generate_cache_reader(size_t offset, size_t req_size) {
}
}
io::FileReaderSPtr cache_reader;
- RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cache_file, &cache_reader, nullptr));
+ RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cache_file, &cache_reader));
_cache_file_readers.emplace(offset, cache_reader);
_last_match_times.emplace(offset, time(nullptr));
LOG(INFO) << "Create cache file from remote file successfully: "
diff --git a/be/src/io/cache/sub_file_cache.h b/be/src/io/cache/sub_file_cache.h
index f8a1a48ba5..4766a4b6a6 100644
--- a/be/src/io/cache/sub_file_cache.h
+++ b/be/src/io/cache/sub_file_cache.h
@@ -36,11 +36,6 @@ public:
Status close() override { return _remote_file_reader->close(); }
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
- Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read);
-
const Path& path() const override { return _remote_file_reader->path(); }
size_t size() const override { return _remote_file_reader->size(); }
@@ -65,6 +60,10 @@ public:
FileSystemSPtr fs() const override { return _remote_file_reader->fs(); }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
Status _generate_cache_reader(size_t offset, size_t req_size);
diff --git a/be/src/io/cache/whole_file_cache.cpp b/be/src/io/cache/whole_file_cache.cpp
index aa98df1501..5f4935cd16 100644
--- a/be/src/io/cache/whole_file_cache.cpp
+++ b/be/src/io/cache/whole_file_cache.cpp
@@ -36,28 +36,18 @@ WholeFileCache::WholeFileCache(const Path& cache_dir, int64_t alive_time_sec,
WholeFileCache::~WholeFileCache() {}
-Status WholeFileCache::read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
- if (bthread_self() == 0) {
- return read_at_impl(offset, result, io_ctx, bytes_read);
- }
- Status s;
- auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status WholeFileCache::read_at_impl(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
- if (io_ctx.reader_type != READER_QUERY) {
- return _remote_file_reader->read_at(offset, result, io_ctx, bytes_read);
+Status WholeFileCache::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) {
+ DCHECK(io_ctx);
+ if (io_ctx->reader_type != READER_QUERY) {
+ return _remote_file_reader->read_at(offset, result, bytes_read, io_ctx);
}
if (_cache_file_reader == nullptr) {
RETURN_IF_ERROR(_generate_cache_reader(offset, result.size));
}
std::shared_lock<std::shared_mutex> rlock(_cache_lock);
RETURN_NOT_OK_STATUS_WITH_WARN(
- _cache_file_reader->read_at(offset, result, io_ctx, bytes_read),
+ _cache_file_reader->read_at(offset, result, bytes_read, io_ctx),
fmt::format("Read local cache file failed: {}", _cache_file_reader->path().native()));
if (*bytes_read != result.size) {
LOG(ERROR) << "read cache file failed: " << _cache_file_reader->path().native()
@@ -137,8 +127,7 @@ Status WholeFileCache::_generate_cache_reader(size_t offset, size_t req_size) {
return st;
}
}
- RETURN_IF_ERROR(
- io::global_local_filesystem()->open_file(cache_file, &_cache_file_reader, nullptr));
+ RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cache_file, &_cache_file_reader));
_cache_file_size = _cache_file_reader->size();
LOG(INFO) << "Create cache file from remote file successfully: "
<< _remote_file_reader->path().native() << " -> " << cache_file.native();
diff --git a/be/src/io/cache/whole_file_cache.h b/be/src/io/cache/whole_file_cache.h
index 556827f67c..0036678114 100644
--- a/be/src/io/cache/whole_file_cache.h
+++ b/be/src/io/cache/whole_file_cache.h
@@ -36,11 +36,6 @@ public:
Status close() override { return _remote_file_reader->close(); }
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
- Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read);
-
const Path& path() const override { return _remote_file_reader->path(); }
size_t size() const override { return _remote_file_reader->size(); }
@@ -63,6 +58,10 @@ public:
FileSystemSPtr fs() const override { return _remote_file_reader->fs(); }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
Status _generate_cache_reader(size_t offset, size_t req_size);
diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp
index d17faec53b..09ee20d8c6 100644
--- a/be/src/io/file_factory.cpp
+++ b/be/src/io/file_factory.cpp
@@ -19,21 +19,21 @@
#include "common/config.h"
#include "common/status.h"
-#include "io/broker_writer.h"
#include "io/fs/broker_file_system.h"
+#include "io/fs/broker_file_writer.h"
#include "io/fs/file_reader_options.h"
#include "io/fs/file_system.h"
#include "io/fs/hdfs_file_system.h"
+#include "io/fs/hdfs_file_writer.h"
#include "io/fs/local_file_system.h"
+#include "io/fs/local_file_writer.h"
#include "io/fs/remote_file_system.h"
#include "io/fs/s3_file_system.h"
-#include "io/hdfs_writer.h"
-#include "io/local_file_writer.h"
-#include "io/s3_writer.h"
-#include "olap/iterators.h"
+#include "io/fs/s3_file_writer.h"
#include "runtime/exec_env.h"
#include "runtime/stream_load/new_load_stream_mgr.h"
#include "runtime/stream_load/stream_load_context.h"
+#include "util/s3_uri.h"
namespace doris {
@@ -41,23 +41,34 @@ Status FileFactory::create_file_writer(TFileType::type type, ExecEnv* env,
const std::vector<TNetworkAddress>& broker_addresses,
const std::map<std::string, std::string>& properties,
const std::string& path, int64_t start_offset,
- std::unique_ptr<FileWriter>& file_writer) {
+ std::unique_ptr<io::FileWriter>& file_writer) {
switch (type) {
case TFileType::FILE_LOCAL: {
- file_writer.reset(new LocalFileWriter(path, start_offset));
+ RETURN_IF_ERROR(io::global_local_filesystem()->create_file(path, &file_writer));
break;
}
case TFileType::FILE_BROKER: {
- file_writer.reset(new BrokerWriter(env, broker_addresses, properties, path, start_offset));
+ std::shared_ptr<io::BrokerFileSystem> fs;
+ RETURN_IF_ERROR(io::BrokerFileSystem::create(broker_addresses[0], properties, 0, &fs));
+ RETURN_IF_ERROR(fs->create_file(path, &file_writer));
break;
}
case TFileType::FILE_S3: {
- file_writer.reset(new S3Writer(properties, path, start_offset));
+ S3URI s3_uri(path);
+ RETURN_IF_ERROR(s3_uri.parse());
+ S3Conf s3_conf;
+ RETURN_IF_ERROR(
+ S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf));
+ std::shared_ptr<io::S3FileSystem> fs;
+ RETURN_IF_ERROR(io::S3FileSystem::create(s3_conf, "", &fs));
+ RETURN_IF_ERROR(fs->create_file(path, &file_writer));
break;
}
case TFileType::FILE_HDFS: {
- RETURN_IF_ERROR(create_hdfs_writer(
- const_cast<std::map<std::string, std::string>&>(properties), path, file_writer));
+ THdfsParams hdfs_params = parse_properties(properties);
+ std::shared_ptr<io::HdfsFileSystem> fs;
+ RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs));
+ RETURN_IF_ERROR(fs->create_file(path, &file_writer));
break;
}
default:
@@ -71,34 +82,34 @@ Status FileFactory::create_file_reader(RuntimeProfile* /*profile*/,
const FileSystemProperties& system_properties,
const FileDescription& file_description,
std::shared_ptr<io::FileSystem>* file_system,
- io::FileReaderSPtr* file_reader, IOContext* io_ctx) {
+ io::FileReaderSPtr* file_reader) {
TFileType::type type = system_properties.system_type;
auto cache_policy = io::FileCachePolicy::NO_CACHE;
- if (config::enable_file_cache && io_ctx->enable_file_cache) {
+ if (config::enable_file_cache) {
cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE;
}
io::FileBlockCachePathPolicy file_block_cache;
io::FileReaderOptions reader_options(cache_policy, file_block_cache);
switch (type) {
case TFileType::FILE_LOCAL: {
- RETURN_IF_ERROR(io::global_local_filesystem()->open_file(
- file_description.path, reader_options, file_reader, io_ctx));
+ RETURN_IF_ERROR(io::global_local_filesystem()->open_file(file_description.path,
+ reader_options, file_reader));
break;
}
case TFileType::FILE_S3: {
RETURN_IF_ERROR(create_s3_reader(system_properties.properties, file_description.path,
- file_system, file_reader, reader_options, io_ctx));
+ file_system, file_reader, reader_options));
break;
}
case TFileType::FILE_HDFS: {
RETURN_IF_ERROR(create_hdfs_reader(system_properties.hdfs_params, file_description.path,
- file_system, file_reader, reader_options, io_ctx));
+ file_system, file_reader, reader_options));
break;
}
case TFileType::FILE_BROKER: {
RETURN_IF_ERROR(create_broker_reader(system_properties.broker_addresses[0],
system_properties.properties, file_description,
- file_system, file_reader, reader_options, io_ctx));
+ file_system, file_reader, reader_options));
break;
}
default:
@@ -120,18 +131,11 @@ Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderS
Status FileFactory::create_hdfs_reader(const THdfsParams& hdfs_params, const std::string& path,
std::shared_ptr<io::FileSystem>* hdfs_file_system,
io::FileReaderSPtr* reader,
- const io::FileReaderOptions& reader_options,
- IOContext* io_ctx) {
- *hdfs_file_system = io::HdfsFileSystem::create(hdfs_params, "");
- RETURN_IF_ERROR((std::static_pointer_cast<io::HdfsFileSystem>(*hdfs_file_system))->connect());
- RETURN_IF_ERROR((*hdfs_file_system)->open_file(path, reader_options, reader, io_ctx));
- return Status::OK();
-}
-
-Status FileFactory::create_hdfs_writer(const std::map<std::string, std::string>& properties,
- const std::string& path,
- std::unique_ptr<FileWriter>& writer) {
- writer.reset(new HDFSWriter(properties, path));
+ const io::FileReaderOptions& reader_options) {
+ std::shared_ptr<io::HdfsFileSystem> fs;
+ RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs));
+ RETURN_IF_ERROR(fs->open_file(path, reader_options, reader));
+ *hdfs_file_system = std::move(fs);
return Status::OK();
}
@@ -139,18 +143,15 @@ Status FileFactory::create_s3_reader(const std::map<std::string, std::string>& p
const std::string& path,
std::shared_ptr<io::FileSystem>* s3_file_system,
io::FileReaderSPtr* reader,
- const io::FileReaderOptions& reader_options,
- IOContext* io_ctx) {
+ const io::FileReaderOptions& reader_options) {
S3URI s3_uri(path);
- if (!s3_uri.parse()) {
- return Status::InvalidArgument("s3 uri is invalid: {}", path);
- }
+ RETURN_IF_ERROR(s3_uri.parse());
S3Conf s3_conf;
- RETURN_IF_ERROR(ClientFactory::convert_properties_to_s3_conf(prop, s3_uri, &s3_conf));
- std::shared_ptr<io::RemoteFileSystem> tmp_fs = io::S3FileSystem::create(std::move(s3_conf), "");
- RETURN_IF_ERROR(tmp_fs->connect());
- RETURN_IF_ERROR(tmp_fs->open_file(s3_uri.get_key(), reader_options, reader, io_ctx));
- *s3_file_system = std::move(tmp_fs);
+ RETURN_IF_ERROR(S3ClientFactory::convert_properties_to_s3_conf(prop, s3_uri, &s3_conf));
+ std::shared_ptr<io::S3FileSystem> fs;
+ RETURN_IF_ERROR(io::S3FileSystem::create(std::move(s3_conf), "", &fs));
+ RETURN_IF_ERROR(fs->open_file(path, reader_options, reader));
+ *s3_file_system = std::move(fs);
return Status::OK();
}
@@ -159,14 +160,12 @@ Status FileFactory::create_broker_reader(const TNetworkAddress& broker_addr,
const FileDescription& file_description,
std::shared_ptr<io::FileSystem>* broker_file_system,
io::FileReaderSPtr* reader,
- const io::FileReaderOptions& reader_options,
- IOContext* io_ctx) {
- *broker_file_system =
- io::BrokerFileSystem::create(broker_addr, prop, file_description.file_size);
+ const io::FileReaderOptions& reader_options) {
+ std::shared_ptr<io::BrokerFileSystem> fs;
RETURN_IF_ERROR(
- (std::static_pointer_cast<io::BrokerFileSystem>(*broker_file_system))->connect());
- RETURN_IF_ERROR((*broker_file_system)
- ->open_file(file_description.path, reader_options, reader, io_ctx));
+ io::BrokerFileSystem::create(broker_addr, prop, file_description.file_size, &fs));
+ RETURN_IF_ERROR(fs->open_file(file_description.path, reader_options, reader));
+ *broker_file_system = std::move(fs);
return Status::OK();
}
} // namespace doris
diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h
index 413795b7cd..2a7a2fe9b4 100644
--- a/be/src/io/file_factory.h
+++ b/be/src/io/file_factory.h
@@ -18,8 +18,8 @@
#include "gen_cpp/PlanNodes_types.h"
#include "gen_cpp/Types_types.h"
-#include "io/file_writer.h"
#include "io/fs/file_reader.h"
+#include "io/fs/file_writer.h"
namespace doris {
namespace io {
@@ -45,18 +45,19 @@ struct FileDescription {
class FileFactory {
public:
- // Create FileWriter
+ /// Create FileWriter
static Status create_file_writer(TFileType::type type, ExecEnv* env,
const std::vector<TNetworkAddress>& broker_addresses,
const std::map<std::string, std::string>& properties,
const std::string& path, int64_t start_offset,
- std::unique_ptr<FileWriter>& file_writer);
+ std::unique_ptr<io::FileWriter>& file_writer);
+ /// Create FileReader
static Status create_file_reader(RuntimeProfile* profile,
const FileSystemProperties& system_properties,
const FileDescription& file_description,
std::shared_ptr<io::FileSystem>* file_system,
- io::FileReaderSPtr* file_reader, IOContext* io_ctx);
+ io::FileReaderSPtr* file_reader);
// Create FileReader for stream load pipe
static Status create_pipe_reader(const TUniqueId& load_id, io::FileReaderSPtr* file_reader);
@@ -64,25 +65,20 @@ public:
static Status create_hdfs_reader(const THdfsParams& hdfs_params, const std::string& path,
std::shared_ptr<io::FileSystem>* hdfs_file_system,
io::FileReaderSPtr* reader,
- const io::FileReaderOptions& reader_options,
- IOContext* io_ctx);
-
- static Status create_hdfs_writer(const std::map<std::string, std::string>& properties,
- const std::string& path, std::unique_ptr<FileWriter>& writer);
+ const io::FileReaderOptions& reader_options);
static Status create_s3_reader(const std::map<std::string, std::string>& prop,
const std::string& path,
std::shared_ptr<io::FileSystem>* s3_file_system,
io::FileReaderSPtr* reader,
- const io::FileReaderOptions& reader_options, IOContext* io_ctx);
+ const io::FileReaderOptions& reader_options);
static Status create_broker_reader(const TNetworkAddress& broker_addr,
const std::map<std::string, std::string>& prop,
const FileDescription& file_description,
std::shared_ptr<io::FileSystem>* hdfs_file_system,
io::FileReaderSPtr* reader,
- const io::FileReaderOptions& reader_options,
- IOContext* io_ctx);
+ const io::FileReaderOptions& reader_options);
static TFileType::type convert_storage_type(TStorageBackendType::type type) {
switch (type) {
diff --git a/be/src/io/fs/broker_file_reader.cpp b/be/src/io/fs/broker_file_reader.cpp
index b708627660..40ddfe6b44 100644
--- a/be/src/io/fs/broker_file_reader.cpp
+++ b/be/src/io/fs/broker_file_reader.cpp
@@ -77,8 +77,8 @@ Status BrokerFileReader::close() {
return Status::OK();
}
-Status BrokerFileReader::read_at(size_t offset, Slice result, const IOContext& /*io_ctx*/,
- size_t* bytes_read) {
+Status BrokerFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) {
DCHECK(!closed());
size_t bytes_req = result.size;
char* to = result.data;
diff --git a/be/src/io/fs/broker_file_reader.h b/be/src/io/fs/broker_file_reader.h
index 5a753462bd..288c5d25d4 100644
--- a/be/src/io/fs/broker_file_reader.h
+++ b/be/src/io/fs/broker_file_reader.h
@@ -39,9 +39,6 @@ public:
Status close() override;
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
const Path& path() const override { return _path; }
size_t size() const override { return _file_size; }
@@ -50,6 +47,10 @@ public:
FileSystemSPtr fs() const override { return _fs; }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
const Path& _path;
size_t _file_size;
diff --git a/be/src/io/fs/broker_file_system.cpp b/be/src/io/fs/broker_file_system.cpp
index 681fa014d4..f962cdb810 100644
--- a/be/src/io/fs/broker_file_system.cpp
+++ b/be/src/io/fs/broker_file_system.cpp
@@ -21,10 +21,11 @@
#include <gen_cpp/TPaloBrokerService.h>
#include "io/fs/broker_file_reader.h"
+#include "io/fs/broker_file_writer.h"
+#include "io/fs/local_file_system.h"
#include "runtime/broker_mgr.h"
#include "runtime/exec_env.h"
#include "util/defer_op.h"
-#include "util/storage_backend.h"
namespace doris {
namespace io {
@@ -50,17 +51,17 @@ inline const std::string& client_id(const TNetworkAddress& addr) {
#endif
#ifndef CHECK_BROKER_CLIENT
-#define CHECK_BROKER_CLIENT(client) \
- if (!client) { \
- return Status::InternalError("init Broker client error"); \
+#define CHECK_BROKER_CLIENT(client) \
+ if (!client) { \
+ return Status::IOError("init Broker client error"); \
}
#endif
-std::shared_ptr<BrokerFileSystem> BrokerFileSystem::create(
- const TNetworkAddress& broker_addr, const std::map<std::string, std::string>& broker_prop,
- size_t file_size) {
- return std::shared_ptr<BrokerFileSystem>(
- new BrokerFileSystem(broker_addr, broker_prop, file_size));
+Status BrokerFileSystem::create(const TNetworkAddress& broker_addr,
+ const std::map<std::string, std::string>& broker_prop,
+ size_t file_size, std::shared_ptr<BrokerFileSystem>* fs) {
+ (*fs).reset(new BrokerFileSystem(broker_addr, broker_prop, file_size));
+ return (*fs)->connect();
}
BrokerFileSystem::BrokerFileSystem(const TNetworkAddress& broker_addr,
@@ -71,25 +72,24 @@ BrokerFileSystem::BrokerFileSystem(const TNetworkAddress& broker_addr,
_broker_prop(broker_prop),
_file_size(file_size) {}
-Status BrokerFileSystem::connect() {
+Status BrokerFileSystem::connect_impl() {
Status status = Status::OK();
_client.reset(new BrokerServiceConnection(client_cache(), _broker_addr,
config::thrift_rpc_timeout_ms, &status));
- if (!status.ok()) {
- std::stringstream ss;
- ss << "failed to get broker client. "
- << "broker addr: " << _broker_addr << ". msg: " << status;
- status = Status::InternalError(ss.str());
- }
return status;
}
-Status BrokerFileSystem::open_file(const Path& path, FileReaderSPtr* reader,
- IOContext* /*io_ctx*/) {
+Status BrokerFileSystem::create_file_impl(const Path& path, FileWriterPtr* writer) {
+ *writer = std::make_unique<BrokerFileWriter>(ExecEnv::GetInstance(), _broker_addr, _broker_prop,
+ path, 0 /* offset */, getSPtr());
+ return Status::OK();
+}
+
+Status BrokerFileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader) {
CHECK_BROKER_CLIENT(_client);
TBrokerOpenReaderRequest request;
request.__set_version(TBrokerVersion::VERSION_ONE);
- request.__set_path(path);
+ request.__set_path(file);
request.__set_startOffset(0);
request.__set_clientId(client_id(_broker_addr));
request.__set_properties(_broker_prop);
@@ -105,39 +105,31 @@ Status BrokerFileSystem::open_file(const Path& path, FileReaderSPtr* reader,
(*_client)->openReader(*response, request);
}
} catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "Open broker reader failed, broker:" << _broker_addr << " failed: " << e.what();
- return Status::RpcError(ss.str());
+ return Status::RpcError("failed to open file {}: {}", file.native(), error_msg(e.what()));
}
if (response->opStatus.statusCode != TBrokerOperationStatusCode::OK) {
- std::stringstream ss;
- ss << "Open broker reader failed, broker: " << _broker_addr
- << " failed: " << response->opStatus.message;
- return Status::InternalError(ss.str());
- }
- // TODO(cmy): The file size is no longer got from openReader() method.
- // But leave the code here for compatibility.
- // This will be removed later.
- TBrokerFD fd;
- if (response->__isset.size) {
- _file_size = response->size;
+ return Status::IOError("failed to open file {}: {}", file.native(),
+ error_msg(response->opStatus.message));
}
- fd = response->fd;
*reader = std::make_shared<BrokerFileReader>(
- _broker_addr, path, _file_size, fd,
+ _broker_addr, file, _file_size, response->fd,
std::static_pointer_cast<BrokerFileSystem>(shared_from_this()));
return Status::OK();
}
-Status BrokerFileSystem::delete_file(const Path& path) {
+Status BrokerFileSystem::create_directory_impl(const Path& /*path*/) {
+ return Status::NotSupported("create directory not implemented!");
+}
+
+Status BrokerFileSystem::delete_file_impl(const Path& file) {
CHECK_BROKER_CLIENT(_client);
try {
// rm file from remote path
TBrokerDeletePathRequest del_req;
TBrokerOperationStatus del_rep;
del_req.__set_version(TBrokerVersion::VERSION_ONE);
- del_req.__set_path(path);
+ del_req.__set_path(file);
del_req.__set_properties(_broker_prop);
try {
@@ -150,27 +142,27 @@ Status BrokerFileSystem::delete_file(const Path& path) {
if (del_rep.statusCode == TBrokerOperationStatusCode::OK) {
return Status::OK();
} else {
- std::stringstream ss;
- ss << "failed to delete from remote path: " << path << ", msg: " << del_rep.message;
- return Status::InternalError(ss.str());
+ return Status::IOError("failed to delete file {}: {}", file.native(),
+ error_msg(del_rep.message));
}
} catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "failed to delete file in remote path: " << path << ", msg: " << e.what();
- return Status::RpcError(ss.str());
+ return Status::RpcError("failed to delete file {}: {}", file.native(), error_msg(e.what()));
}
}
-Status BrokerFileSystem::create_directory(const Path& /*path*/) {
- return Status::NotSupported("create directory not implemented!");
+// Delete all files under path.
+Status BrokerFileSystem::delete_directory_impl(const Path& dir) {
+ return delete_file_impl(dir);
}
-// Delete all files under path.
-Status BrokerFileSystem::delete_directory(const Path& path) {
- return delete_file(path);
+Status BrokerFileSystem::batch_delete_impl(const std::vector<Path>& files) {
+ for (auto& file : files) {
+ RETURN_IF_ERROR(delete_file_impl(file));
+ }
+ return Status::OK();
}
-Status BrokerFileSystem::exists(const Path& path, bool* res) const {
+Status BrokerFileSystem::exists_impl(const Path& path, bool* res) const {
CHECK_BROKER_CLIENT(_client);
*res = false;
try {
@@ -188,29 +180,32 @@ Status BrokerFileSystem::exists(const Path& path, bool* res) const {
}
if (check_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
- std::stringstream ss;
- ss << "failed to check exist: " << path << ", msg: " << check_rep.opStatus.message;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
+ return Status::IOError("failed to check exist of path {}: {}", path.native(),
+ error_msg(check_rep.opStatus.message));
} else if (!check_rep.isPathExist) {
- return Status::NotFound("{} not exists!", path.string());
+ *res = false;
+ return Status::OK();
} else {
*res = true;
return Status::OK();
}
} catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "failed to check exist: " << path << ", msg: " << e.what();
- return Status::RpcError(ss.str());
+ return Status::RpcError("failed to check exist of path {}: {}", path.native(),
+ error_msg(e.what()));
}
}
-Status BrokerFileSystem::file_size(const Path& path, size_t* file_size) const {
+Status BrokerFileSystem::file_size_impl(const Path& path, size_t* file_size) const {
*file_size = _file_size;
return Status::OK();
}
-Status BrokerFileSystem::list(const Path& path, std::vector<Path>* files) {
+Status BrokerFileSystem::list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) {
+ RETURN_IF_ERROR(exists_impl(dir, exists));
+ if (!(*exists)) {
+ return Status::OK();
+ }
CHECK_BROKER_CLIENT(_client);
Status status = Status::OK();
try {
@@ -218,7 +213,7 @@ Status BrokerFileSystem::list(const Path& path, std::vector<Path>* files) {
TBrokerListResponse list_rep;
TBrokerListPathRequest list_req;
list_req.__set_version(TBrokerVersion::VERSION_ONE);
- list_req.__set_path(path / "*");
+ list_req.__set_path(dir / "*");
list_req.__set_isRecursive(false);
list_req.__set_properties(_broker_prop);
list_req.__set_fileNameOnly(true); // we only need file name, not abs path
@@ -231,52 +226,204 @@ Status BrokerFileSystem::list(const Path& path, std::vector<Path>* files) {
}
if (list_rep.opStatus.statusCode == TBrokerOperationStatusCode::FILE_NOT_FOUND) {
- LOG(INFO) << "path does not exist: " << path;
+ LOG(INFO) << "path does not exist: " << dir;
+ *exists = false;
return Status::OK();
} else if (list_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
- std::stringstream ss;
- ss << "failed to list files from remote path: " << path
- << ", msg: " << list_rep.opStatus.message;
- return Status::InternalError(ss.str());
+ return Status::IOError("failed to list dir {}: {}", dir.native(),
+ error_msg(list_rep.opStatus.message));
}
LOG(INFO) << "finished to list files from remote path. file num: " << list_rep.files.size();
+ *exists = true;
// split file name and checksum
for (const auto& file : list_rep.files) {
- if (file.isDir) {
+ if (only_file && file.isDir) {
// this is not a file
continue;
}
-
- const std::string& file_name = file.path;
- size_t pos = file_name.find_last_of('.');
- if (pos == std::string::npos || pos == file_name.size() - 1) {
- // Not found checksum separator, ignore this file
- continue;
- }
-
- FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1),
- file.size};
- files->emplace_back(std::string(file_name, 0, pos));
- VLOG(2) << "split remote file: " << std::string(file_name, 0, pos)
- << ", checksum: " << std::string(file_name, pos + 1);
+ FileInfo file_info;
+ file_info.file_name = file.path;
+ file_info.file_size = file.size;
+ file_info.is_file = !file.isDir;
+ files->emplace_back(std::move(file_info));
}
LOG(INFO) << "finished to split files. valid file num: " << files->size();
-
} catch (apache::thrift::TException& e) {
std::stringstream ss;
- ss << "failed to list files in remote path: " << path << ", msg: " << e.what();
- return Status::RpcError(ss.str());
+ ss << "failed to list files in remote path: " << dir << ", msg: " << e.what();
+ return Status::RpcError("failed to list dir {}: {}", dir.native(), error_msg(e.what()));
}
return status;
}
+Status BrokerFileSystem::rename_impl(const Path& orig_name, const Path& new_name) {
+ CHECK_BROKER_CLIENT(_client);
+ try {
+ TBrokerOperationStatus op_status;
+ TBrokerRenamePathRequest rename_req;
+ rename_req.__set_version(TBrokerVersion::VERSION_ONE);
+ rename_req.__set_srcPath(orig_name);
+ rename_req.__set_destPath(new_name);
+ rename_req.__set_properties(_broker_prop);
+
+ try {
+ (*_client)->renamePath(op_status, rename_req);
+ } catch (apache::thrift::transport::TTransportException& e) {
+ RETURN_IF_ERROR((*_client).reopen());
+ (*_client)->renamePath(op_status, rename_req);
+ }
+
+ if (op_status.statusCode != TBrokerOperationStatusCode::OK) {
+ return Status::IOError("failed to rename from {} to {}: {}", orig_name.native(),
+ new_name.native(), error_msg(op_status.message));
+ }
+ } catch (apache::thrift::TException& e) {
+ return Status::RpcError("failed to rename from {} to {}: {}", orig_name.native(),
+ new_name.native(), error_msg(e.what()));
+ }
+
+ LOG(INFO) << "finished to rename file. orig: " << orig_name << ", new: " << new_name;
+ return Status::OK();
+}
+
+Status BrokerFileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) {
+ return rename_impl(orig_name, new_name);
+}
+
+Status BrokerFileSystem::upload_impl(const Path& local_file, const Path& remote_file) {
+ // 1. open local file for read
+ FileSystemSPtr local_fs = global_local_filesystem();
+ FileReaderSPtr local_reader = nullptr;
+ RETURN_IF_ERROR(local_fs->open_file(local_file, &local_reader));
+
+ size_t file_len = local_reader->size();
+ if (file_len == -1) {
+ return Status::IOError("failed to get length of file: {}: {}", local_file.native(),
+ error_msg(""));
+ }
+
+ // NOTICE: broker writer must be closed before calling rename
+ FileWriterPtr broker_writer = nullptr;
+ RETURN_IF_ERROR(create_file_impl(remote_file, &broker_writer));
+
+ constexpr size_t buf_sz = 1024 * 1024;
+ char read_buf[buf_sz];
+ size_t left_len = file_len;
+ size_t read_offset = 0;
+ size_t bytes_read = 0;
+ while (left_len > 0) {
+ size_t read_len = left_len > buf_sz ? buf_sz : left_len;
+ RETURN_IF_ERROR(local_reader->read_at(read_offset, {read_buf, read_len}, &bytes_read));
+ // write through broker
+ RETURN_IF_ERROR(broker_writer->append({read_buf, read_len}));
+
+ read_offset += read_len;
+ left_len -= read_len;
+ }
+
+ // close manually, because we need to check its close status
+ RETURN_IF_ERROR(broker_writer->close());
+ LOG(INFO) << "finished to write file via broker. file: " << local_file
+ << ", length: " << file_len;
+ return Status::OK();
+}
+
+Status BrokerFileSystem::batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) {
+ DCHECK(local_files.size() == remote_files.size());
+ for (int i = 0; i < local_files.size(); ++i) {
+ RETURN_IF_ERROR(upload_impl(local_files[i], remote_files[i]));
+ }
+ return Status::OK();
+}
+
+Status BrokerFileSystem::direct_upload_impl(const Path& remote_file, const std::string& content) {
+ FileWriterPtr broker_writer = nullptr;
+ RETURN_IF_ERROR(create_file_impl(remote_file, &broker_writer));
+ RETURN_IF_ERROR(broker_writer->append({content}));
+ return broker_writer->close();
+}
+
+Status BrokerFileSystem::upload_with_checksum_impl(const Path& local_file, const Path& remote_file,
+ const std::string& checksum) {
+ std::string temp = remote_file.string() + ".part";
+ std::string final_file = remote_file.string() + "." + checksum;
+ RETURN_IF_ERROR(upload_impl(local_file, temp));
+ return rename_impl(temp, final_file);
+}
+
+Status BrokerFileSystem::download_impl(const Path& remote_file, const Path& local_file) {
+ // 1. open remote file for read
+ FileReaderSPtr broker_reader = nullptr;
+ RETURN_IF_ERROR(open_file_internal(remote_file, &broker_reader));
+
+ // 2. remove the existing local file if exist
+ if (std::filesystem::remove(local_file)) {
+ VLOG(2) << "remove the previously exist local file: " << local_file;
+ }
+
+ // 3. open local file for write
+ FileSystemSPtr local_fs = global_local_filesystem();
+ FileWriterPtr local_writer = nullptr;
+ RETURN_IF_ERROR(local_fs->create_file(local_file, &local_writer));
+
+ // 4. read remote and write to local
+ VLOG(2) << "read remote file: " << remote_file << " to local: " << local_file;
+ constexpr size_t buf_sz = 1024 * 1024;
+ std::unique_ptr<uint8_t[]> read_buf(new uint8_t[buf_sz]);
+ size_t write_offset = 0;
+ size_t cur_offset = 0;
+ while (true) {
+ size_t read_len = 0;
+ Slice file_slice(read_buf.get(), buf_sz);
+ RETURN_IF_ERROR(broker_reader->read_at(cur_offset, file_slice, &read_len));
+ cur_offset += read_len;
+ if (read_len == 0) {
+ break;
+ }
+
+ RETURN_IF_ERROR(local_writer->write_at(write_offset, {read_buf.get(), read_len}));
+ write_offset += read_len;
+ } // file_handler should be closed before calculating checksum
+
+ return Status::OK();
+}
+
+Status BrokerFileSystem::direct_download_impl(const Path& remote_impl, std::string* content) {
+ // 1. open remote file for read
+ FileReaderSPtr broker_reader = nullptr;
+ RETURN_IF_ERROR(open_file_internal(remote_impl, &broker_reader));
+
+ constexpr size_t buf_sz = 1024 * 1024;
+ std::unique_ptr<char[]> read_buf(new char[buf_sz]);
+ size_t write_offset = 0;
+ size_t cur_offset = 0;
+ while (true) {
+ size_t read_len = 0;
+ Slice file_slice(read_buf.get(), buf_sz);
+ RETURN_IF_ERROR(broker_reader->read_at(cur_offset, file_slice, &read_len));
+ cur_offset += read_len;
+ if (read_len == 0) {
+ break;
+ }
+
+ content->insert(write_offset, read_buf.get(), read_len);
+ write_offset += read_len;
+ }
+ return Status::OK();
+}
+
Status BrokerFileSystem::get_client(std::shared_ptr<BrokerServiceConnection>* client) const {
CHECK_BROKER_CLIENT(_client);
*client = _client;
return Status::OK();
}
+std::string BrokerFileSystem::error_msg(const std::string& err) const {
+ return fmt::format("({}:{}), {}", _broker_addr.hostname, _broker_addr.port, err);
+}
+
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/broker_file_system.h b/be/src/io/fs/broker_file_system.h
index bf55d49a53..0328add3ca 100644
--- a/be/src/io/fs/broker_file_system.h
+++ b/be/src/io/fs/broker_file_system.h
@@ -24,51 +24,42 @@ namespace doris {
namespace io {
class BrokerFileSystem final : public RemoteFileSystem {
public:
- static std::shared_ptr<BrokerFileSystem> create(
- const TNetworkAddress& broker_addr,
- const std::map<std::string, std::string>& broker_prop, size_t file_size);
+ static Status create(const TNetworkAddress& broker_addr,
+ const std::map<std::string, std::string>& broker_prop, size_t file_size,
+ std::shared_ptr<BrokerFileSystem>* fs);
~BrokerFileSystem() override = default;
- Status create_file(const Path& /*path*/, FileWriterPtr* /*writer*/) override {
- return Status::NotSupported("Currently not support to create file through broker.");
- }
-
- Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override;
-
- Status delete_file(const Path& path) override;
-
- Status create_directory(const Path& path) override;
-
- // Delete all files under path.
- Status delete_directory(const Path& path) override;
-
- Status link_file(const Path& /*src*/, const Path& /*dest*/) override {
- return Status::NotSupported("Not supported link file through broker.");
- }
-
- Status exists(const Path& path, bool* res) const override;
-
- Status file_size(const Path& path, size_t* file_size) const override;
-
- Status list(const Path& path, std::vector<Path>* files) override;
-
- Status upload(const Path& /*local_path*/, const Path& /*dest_path*/) override {
- return Status::NotSupported("Currently not support to upload file to HDFS");
- }
-
- Status batch_upload(const std::vector<Path>& /*local_paths*/,
- const std::vector<Path>& /*dest_paths*/) override {
- return Status::NotSupported("Currently not support to batch upload file to HDFS");
- }
-
- Status connect() override;
-
Status get_client(std::shared_ptr<BrokerServiceConnection>* client) const;
+protected:
+ Status connect_impl() override;
+ Status create_file_impl(const Path& file, FileWriterPtr* writer) override;
+ Status open_file_internal(const Path& file, FileReaderSPtr* reader) override;
+ Status create_directory_impl(const Path& dir) override;
+ Status delete_file_impl(const Path& file) override;
+ Status delete_directory_impl(const Path& dir) override;
+ Status batch_delete_impl(const std::vector<Path>& files) override;
+ Status exists_impl(const Path& path, bool* res) const override;
+ Status file_size_impl(const Path& file, size_t* file_size) const override;
+ Status list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) override;
+ Status rename_impl(const Path& orig_name, const Path& new_name) override;
+ Status rename_dir_impl(const Path& orig_name, const Path& new_name) override;
+
+ Status upload_impl(const Path& local_file, const Path& remote_file) override;
+ Status batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) override;
+ Status direct_upload_impl(const Path& remote_file, const std::string& content) override;
+ Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file,
+ const std::string& checksum) override;
+ Status download_impl(const Path& remote_file, const Path& local_file) override;
+ Status direct_download_impl(const Path& remote_file, std::string* content) override;
+
private:
BrokerFileSystem(const TNetworkAddress& broker_addr,
const std::map<std::string, std::string>& broker_prop, size_t file_size);
+ std::string error_msg(const std::string& err) const;
const TNetworkAddress& _broker_addr;
const std::map<std::string, std::string>& _broker_prop;
diff --git a/be/src/io/broker_writer.cpp b/be/src/io/fs/broker_file_writer.cpp
similarity index 68%
rename from be/src/io/broker_writer.cpp
rename to be/src/io/fs/broker_file_writer.cpp
index 398b172c07..463c44946f 100644
--- a/be/src/io/broker_writer.cpp
+++ b/be/src/io/fs/broker_file_writer.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "io/broker_writer.h"
+#include "io/fs/broker_file_writer.h"
#include <thrift/protocol/TDebugProtocol.h>
@@ -29,20 +29,22 @@
#include "runtime/exec_env.h"
namespace doris {
-
-BrokerWriter::BrokerWriter(ExecEnv* env, const std::vector<TNetworkAddress>& broker_addresses,
- const std::map<std::string, std::string>& properties,
- const std::string& path, int64_t start_offset)
- : _env(env),
- _addresses(broker_addresses),
+namespace io {
+
+BrokerFileWriter::BrokerFileWriter(ExecEnv* env, const TNetworkAddress& broker_address,
+ const std::map<std::string, std::string>& properties,
+ const std::string& path, int64_t start_offset, FileSystemSPtr fs)
+ : FileWriter(path, fs),
+ _env(env),
+ _address(broker_address),
_properties(properties),
- _path(path),
- _cur_offset(start_offset),
- _is_closed(false),
- _addr_idx(0) {}
+ _cur_offset(start_offset) {}
-BrokerWriter::~BrokerWriter() {
- close();
+BrokerFileWriter::~BrokerFileWriter() {
+ if (_opened) {
+ close();
+ }
+ CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed;
}
#ifdef BE_TEST
@@ -65,172 +67,199 @@ inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) {
}
#endif
-Status BrokerWriter::open() {
- TBrokerOpenWriterRequest request;
+Status BrokerFileWriter::close() {
+ if (_closed) {
+ return Status::OK();
+ }
+ _closed = true;
- const TNetworkAddress& broker_addr = _addresses[_addr_idx];
+ TBrokerCloseWriterRequest request;
request.__set_version(TBrokerVersion::VERSION_ONE);
- request.__set_path(_path);
- request.__set_openMode(TBrokerOpenMode::APPEND);
- request.__set_clientId(client_id(_env, broker_addr));
- request.__set_properties(_properties);
-
- VLOG_ROW << "debug: send broker open writer request: "
+ request.__set_fd(_fd);
+ VLOG_ROW << "debug: send broker close writer request: "
<< apache::thrift::ThriftDebugString(request).c_str();
- TBrokerOpenWriterResponse response;
+ TBrokerOperationStatus response;
try {
Status status;
- BrokerServiceConnection client(client_cache(_env), broker_addr,
- config::thrift_rpc_timeout_ms, &status);
+ // use 20 second because close may take longer in remote storage, sometimes.
+ // TODO(cmy): optimize this if necessary.
+ BrokerServiceConnection client(client_cache(_env), _address, 20000, &status);
if (!status.ok()) {
- LOG(WARNING) << "Create broker writer client failed. "
- << "broker=" << broker_addr << ", status=" << status;
+ LOG(WARNING) << "Create broker write client failed. broker=" << _address
+ << ", status=" << status;
return status;
}
try {
- client->openWriter(response, request);
+ client->closeWriter(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
- RETURN_IF_ERROR(client.reopen());
- client->openWriter(response, request);
+ LOG(WARNING) << "Close broker writer failed. broker:" << _address
+ << " msg:" << e.what();
+ status = client.reopen();
+ if (!status.ok()) {
+ LOG(WARNING) << "Reopen broker writer failed. broker=" << _address
+ << ", status=" << status;
+ return status;
+ }
+ client->closeWriter(response, request);
}
} catch (apache::thrift::TException& e) {
std::stringstream ss;
- ss << "Open broker writer failed, broker:" << broker_addr << " failed:" << e.what();
+ ss << "Close broker writer failed, broker:" << _address << " msg:" << e.what();
LOG(WARNING) << ss.str();
- return Status::RpcError(ss.str());
+ return Status::InternalError(ss.str());
}
- VLOG_ROW << "debug: send broker open writer response: "
+ VLOG_ROW << "debug: send broker close writer response: "
<< apache::thrift::ThriftDebugString(response).c_str();
- if (response.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
+ if (response.statusCode != TBrokerOperationStatusCode::OK) {
std::stringstream ss;
- ss << "Open broker writer failed, broker:" << broker_addr
- << " failed:" << response.opStatus.message;
+ ss << "Close broker writer failed, broker:" << _address << " msg:" << response.message;
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
+ return Status::OK();
+}
- _fd = response.fd;
+Status BrokerFileWriter::abort() {
+ // TODO: should remove file
return Status::OK();
}
-Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len) {
- if (buf_len == 0) {
- *written_len = 0;
- return Status::OK();
+Status BrokerFileWriter::appendv(const Slice* data, size_t data_cnt) {
+ DCHECK(!_closed);
+ if (!_opened) {
+ RETURN_IF_ERROR(_open());
+ _opened = true;
}
- const TNetworkAddress& broker_addr = _addresses[_addr_idx];
- TBrokerPWriteRequest request;
+ for (size_t i = 0; i < data_cnt; i++) {
+ const Slice& result = data[i];
+ size_t left_bytes = result.size;
+ const char* p = result.data;
+ while (left_bytes > 0) {
+ size_t written_bytes = 0;
+ RETURN_IF_ERROR(_write((const uint8_t*)p, left_bytes, &written_bytes));
+ left_bytes -= written_bytes;
+ p += written_bytes;
+ _bytes_appended += written_bytes;
+ }
+ }
+ return Status::OK();
+}
+
+Status BrokerFileWriter::finalize() {
+ return Status::OK();
+}
+
+Status BrokerFileWriter::_open() {
+ TBrokerOpenWriterRequest request;
+
request.__set_version(TBrokerVersion::VERSION_ONE);
- request.__set_fd(_fd);
- request.__set_offset(_cur_offset);
- request.__set_data(std::string(reinterpret_cast<const char*>(buf), buf_len));
+ request.__set_path(_path);
+ request.__set_openMode(TBrokerOpenMode::APPEND);
+ request.__set_clientId(client_id(_env, _address));
+ request.__set_properties(_properties);
- VLOG_ROW << "debug: send broker pwrite request: "
+ VLOG_ROW << "debug: send broker open writer request: "
<< apache::thrift::ThriftDebugString(request).c_str();
- TBrokerOperationStatus response;
+ TBrokerOpenWriterResponse response;
try {
Status status;
- BrokerServiceConnection client(client_cache(_env), broker_addr,
- config::thrift_rpc_timeout_ms, &status);
+ BrokerServiceConnection client(client_cache(_env), _address, config::thrift_rpc_timeout_ms,
+ &status);
if (!status.ok()) {
- LOG(WARNING) << "Create broker write client failed. "
- << "broker=" << broker_addr << ", status=" << status;
+ LOG(WARNING) << "Create broker writer client failed. "
+ << "broker=" << _address << ", status=" << status;
return status;
}
try {
- client->pwrite(response, request);
+ client->openWriter(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
RETURN_IF_ERROR(client.reopen());
- // broker server will check write offset, so it is safe to re-try
- client->pwrite(response, request);
+ client->openWriter(response, request);
}
} catch (apache::thrift::TException& e) {
std::stringstream ss;
- ss << "Fail to write to broker, broker:" << broker_addr << " failed:" << e.what();
+ ss << "Open broker writer failed, broker:" << _address << " failed:" << e.what();
LOG(WARNING) << ss.str();
return Status::RpcError(ss.str());
}
- VLOG_ROW << "debug: send broker pwrite response: "
+ VLOG_ROW << "debug: send broker open writer response: "
<< apache::thrift::ThriftDebugString(response).c_str();
- if (response.statusCode != TBrokerOperationStatusCode::OK) {
+ if (response.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
std::stringstream ss;
- ss << "Fail to write to broker, broker:" << broker_addr << " msg:" << response.message;
+ ss << "Open broker writer failed, broker:" << _address
+ << " failed:" << response.opStatus.message;
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
- *written_len = buf_len;
- _cur_offset += buf_len;
-
+ _fd = response.fd;
return Status::OK();
}
-Status BrokerWriter::close() {
- if (_is_closed) {
+Status BrokerFileWriter::_write(const uint8_t* buf, size_t buf_len, size_t* written_bytes) {
+ if (buf_len == 0) {
+ *written_bytes = 0;
return Status::OK();
}
- TBrokerCloseWriterRequest request;
+ TBrokerPWriteRequest request;
request.__set_version(TBrokerVersion::VERSION_ONE);
request.__set_fd(_fd);
+ request.__set_offset(_cur_offset);
+ request.__set_data(std::string(reinterpret_cast<const char*>(buf), buf_len));
- VLOG_ROW << "debug: send broker close writer request: "
+ VLOG_ROW << "debug: send broker pwrite request: "
<< apache::thrift::ThriftDebugString(request).c_str();
- const TNetworkAddress& broker_addr = _addresses[_addr_idx];
TBrokerOperationStatus response;
try {
Status status;
- // use 20 second because close may take longer in remote storage, sometimes.
- // TODO(cmy): optimize this if necessary.
- BrokerServiceConnection client(client_cache(_env), broker_addr, 20000, &status);
+ BrokerServiceConnection client(client_cache(_env), _address, config::thrift_rpc_timeout_ms,
+ &status);
if (!status.ok()) {
- LOG(WARNING) << "Create broker write client failed. broker=" << broker_addr
- << ", status=" << status;
+ LOG(WARNING) << "Create broker write client failed. "
+ << "broker=" << _address << ", status=" << status;
return status;
}
try {
- client->closeWriter(response, request);
+ client->pwrite(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
- LOG(WARNING) << "Close broker writer failed. broker:" << broker_addr
- << " msg:" << e.what();
- status = client.reopen();
- if (!status.ok()) {
- LOG(WARNING) << "Reopen broker writer failed. broker=" << broker_addr
- << ", status=" << status;
- return status;
- }
- client->closeWriter(response, request);
+ RETURN_IF_ERROR(client.reopen());
+ // broker server will check write offset, so it is safe to re-try
+ client->pwrite(response, request);
}
} catch (apache::thrift::TException& e) {
std::stringstream ss;
- ss << "Close broker writer failed, broker:" << broker_addr << " msg:" << e.what();
+ ss << "Fail to write to broker, broker:" << _address << " failed:" << e.what();
LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
+ return Status::RpcError(ss.str());
}
- VLOG_ROW << "debug: send broker close writer response: "
+ VLOG_ROW << "debug: send broker pwrite response: "
<< apache::thrift::ThriftDebugString(response).c_str();
if (response.statusCode != TBrokerOperationStatusCode::OK) {
std::stringstream ss;
- ss << "Close broker writer failed, broker:" << broker_addr << " msg:" << response.message;
+ ss << "Fail to write to broker, broker:" << _address << " msg:" << response.message;
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
- _is_closed = true;
+ *written_bytes = buf_len;
+ _cur_offset += buf_len;
+
return Status::OK();
}
+} // end namespace io
} // end namespace doris
diff --git a/be/src/io/broker_writer.h b/be/src/io/fs/broker_file_writer.h
similarity index 61%
rename from be/src/io/broker_writer.h
rename to be/src/io/fs/broker_file_writer.h
index 1155fd4e9e..5705cc18df 100644
--- a/be/src/io/broker_writer.h
+++ b/be/src/io/fs/broker_file_writer.h
@@ -25,7 +25,7 @@
#include "common/status.h"
#include "gen_cpp/PaloBrokerService_types.h"
#include "gen_cpp/Types_types.h"
-#include "io/file_writer.h"
+#include "io/fs/file_writer.h"
namespace doris {
@@ -33,32 +33,35 @@ class ExecEnv;
class TBrokerRangeDesc;
class TNetworkAddress;
+namespace io {
+
// Reader of broker file
-class BrokerWriter : public FileWriter {
+class BrokerFileWriter : public FileWriter {
public:
- BrokerWriter(ExecEnv* env, const std::vector<TNetworkAddress>& broker_addresses,
- const std::map<std::string, std::string>& properties, const std::string& path,
- int64_t start_offset);
- virtual ~BrokerWriter();
-
- virtual Status open() override;
+ BrokerFileWriter(ExecEnv* env, const TNetworkAddress& broker_address,
+ const std::map<std::string, std::string>& properties, const std::string& path,
+ int64_t start_offset, FileSystemSPtr fs);
+ virtual ~BrokerFileWriter();
- virtual Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override;
+ Status close() override;
+ Status abort() override;
+ Status appendv(const Slice* data, size_t data_cnt) override;
+ Status finalize() override;
+ Status write_at(size_t offset, const Slice& data) override {
+ return Status::NotSupported("not support");
+ }
- virtual Status close() override;
+private:
+ Status _open();
+ Status _write(const uint8_t* buf, size_t buf_len, size_t* written_bytes);
private:
ExecEnv* _env;
- const std::vector<TNetworkAddress>& _addresses;
+ const TNetworkAddress _address;
const std::map<std::string, std::string>& _properties;
- std::string _path;
int64_t _cur_offset;
-
- bool _is_closed;
TBrokerFD _fd;
-
- // TODO: use for retry if one broker down
- int _addr_idx;
};
+} // end namespace io
} // end namespace doris
diff --git a/be/src/io/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
similarity index 97%
rename from be/src/io/buffered_reader.cpp
rename to be/src/io/fs/buffered_reader.cpp
index dfad14c8c0..af88fb4b89 100644
--- a/be/src/io/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "io/buffered_reader.h"
+#include "io/fs/buffered_reader.h"
#include <algorithm>
#include <sstream>
@@ -26,6 +26,7 @@
#include "util/bit_util.h"
namespace doris {
+namespace io {
BufferedFileStreamReader::BufferedFileStreamReader(io::FileReaderSPtr file, uint64_t offset,
uint64_t length, size_t max_buf_size)
@@ -67,8 +68,7 @@ Status BufferedFileStreamReader::read_bytes(const uint8_t** buf, uint64_t offset
while (has_read < to_read) {
size_t loop_read = 0;
Slice resutl(_buf.get() + buf_remaining + has_read, to_read - has_read);
- IOContext io_context;
- RETURN_IF_ERROR(_file->read_at(_buf_end_offset + has_read, resutl, io_context, &loop_read));
+ RETURN_IF_ERROR(_file->read_at(_buf_end_offset + has_read, resutl, &loop_read));
_statistics.read_calls++;
if (loop_read == 0) {
break;
@@ -88,4 +88,5 @@ Status BufferedFileStreamReader::read_bytes(Slice& slice, uint64_t offset) {
return read_bytes((const uint8_t**)&slice.data, offset, slice.size);
}
+} // namespace io
} // namespace doris
diff --git a/be/src/io/buffered_reader.h b/be/src/io/fs/buffered_reader.h
similarity index 98%
rename from be/src/io/buffered_reader.h
rename to be/src/io/fs/buffered_reader.h
index 8bab3b1a88..15f2f524d7 100644
--- a/be/src/io/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -27,6 +27,7 @@
#include "util/runtime_profile.h"
namespace doris {
+namespace io {
/**
* Load all the needed data in underlying buffer, so the caller does not need to prepare the data container.
@@ -78,4 +79,5 @@ private:
size_t _max_buf_size;
};
+} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/file_reader_options.cpp b/be/src/io/fs/file_reader.cpp
similarity index 64%
copy from be/src/io/fs/file_reader_options.cpp
copy to be/src/io/fs/file_reader.cpp
index 0f8f74dbc2..05de1ba499 100644
--- a/be/src/io/fs/file_reader_options.cpp
+++ b/be/src/io/fs/file_reader.cpp
@@ -15,21 +15,24 @@
// specific language governing permissions and limitations
// under the License.
-#include "io/fs/file_reader_options.h"
+#include "io/fs/file_reader.h"
+
+#include "io/fs/file_system.h"
+#include "io/io_common.h"
+#include "util/async_io.h"
namespace doris {
namespace io {
-FileCachePolicy cache_type_from_string(const std::string& type) {
- if (type == "sub_file_cache") {
- return FileCachePolicy::SUB_FILE_CACHE;
- } else if (type == "whole_file_cache") {
- return FileCachePolicy::WHOLE_FILE_CACHE;
- } else if (type == "file_block_cache") {
- return FileCachePolicy::FILE_BLOCK_CACHE;
- } else {
- return FileCachePolicy::NO_CACHE;
+Status FileReader::read_at(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) {
+ if (bthread_self() == 0) {
+ return read_at_impl(offset, result, bytes_read, io_ctx);
}
+ Status s;
+ auto task = [&] { s = read_at_impl(offset, result, bytes_read, io_ctx); };
+ AsyncIO::run_task(task, fs()->type());
+ return s;
}
} // namespace io
diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h
index 542147bb90..c429b36ed3 100644
--- a/be/src/io/fs/file_reader.h
+++ b/be/src/io/fs/file_reader.h
@@ -19,16 +19,16 @@
#include "common/status.h"
#include "gutil/macros.h"
+#include "io/fs/file_reader_writer_fwd.h"
#include "io/fs/path.h"
#include "util/slice.h"
namespace doris {
-struct IOContext;
-
namespace io {
class FileSystem;
+class IOContext;
class FileReader {
public:
@@ -37,10 +37,12 @@ public:
DISALLOW_COPY_AND_ASSIGN(FileReader);
- virtual Status close() = 0;
+ /// If io_ctx is not null,
+ /// the caller must ensure that the IOContext exists during the left cycle of read_at()
+ Status read_at(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx = nullptr);
- virtual Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) = 0;
+ virtual Status close() = 0;
virtual const Path& path() const = 0;
@@ -49,9 +51,11 @@ public:
virtual bool closed() const = 0;
virtual std::shared_ptr<FileSystem> fs() const = 0;
-};
-using FileReaderSPtr = std::shared_ptr<FileReader>;
+protected:
+ virtual Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) = 0;
+};
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/file_reader_options.cpp b/be/src/io/fs/file_reader_options.cpp
index 0f8f74dbc2..f388f51322 100644
--- a/be/src/io/fs/file_reader_options.cpp
+++ b/be/src/io/fs/file_reader_options.cpp
@@ -20,6 +20,9 @@
namespace doris {
namespace io {
+FileReaderOptions FileReaderOptions::DEFAULT =
+ FileReaderOptions(FileCachePolicy::NO_CACHE, NoCachePathPolicy());
+
FileCachePolicy cache_type_from_string(const std::string& type) {
if (type == "sub_file_cache") {
return FileCachePolicy::SUB_FILE_CACHE;
diff --git a/be/src/io/fs/file_reader_options.h b/be/src/io/fs/file_reader_options.h
index f5a6eaa8f3..cef4a54ea3 100644
--- a/be/src/io/fs/file_reader_options.h
+++ b/be/src/io/fs/file_reader_options.h
@@ -72,6 +72,8 @@ public:
FileCachePolicy cache_type;
const CachePathPolicy& path_policy;
+
+ static FileReaderOptions DEFAULT;
};
} // namespace io
diff --git a/be/src/exec/line_reader.h b/be/src/io/fs/file_reader_writer_fwd.h
similarity index 72%
copy from be/src/exec/line_reader.h
copy to be/src/io/fs/file_reader_writer_fwd.h
index 80a8f94977..a0e3078095 100644
--- a/be/src/exec/line_reader.h
+++ b/be/src/io/fs/file_reader_writer_fwd.h
@@ -14,20 +14,19 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/Cache/FileCache_fwd.h
+// and modified by Doris
#pragma once
-
-#include "common/status.h"
-
namespace doris {
+namespace io {
-// This class is used for CSV scanner, to read content line by line
-class LineReader {
-public:
- virtual ~LineReader() = default;
- virtual Status read_line(const uint8_t** ptr, size_t* size, bool* eof) = 0;
+class FileReader;
+class FileWriter;
- virtual void close() = 0;
-};
+using FileReaderSPtr = std::shared_ptr<FileReader>;
+using FileWriterPtr = std::unique_ptr<FileWriter>;
+} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp
new file mode 100644
index 0000000000..212020437a
--- /dev/null
+++ b/be/src/io/fs/file_system.cpp
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/file_system.h"
+
+#include "util/async_io.h"
+
+namespace doris {
+namespace io {
+
+Status FileSystem::create_file(const Path& file, FileWriterPtr* writer) {
+ auto path = absolute_path(file);
+ if (bthread_self() == 0) {
+ return create_file_impl(path, writer);
+ }
+ Status s;
+ auto task = [&] { s = create_file_impl(path, writer); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::open_file(const Path& file, const FileReaderOptions& reader_options,
+ FileReaderSPtr* reader) {
+ auto path = absolute_path(file);
+ if (bthread_self() == 0) {
+ return open_file_impl(path, reader_options, reader);
+ }
+ Status s;
+ auto task = [&] { s = open_file_impl(path, reader_options, reader); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::create_directory(const Path& dir) {
+ auto path = absolute_path(dir);
+ if (bthread_self() == 0) {
+ return create_directory_impl(path);
+ }
+ Status s;
+ auto task = [&] { s = create_directory_impl(path); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::delete_file(const Path& file) {
+ auto path = absolute_path(file);
+ if (bthread_self() == 0) {
+ return delete_file_impl(path);
+ }
+ Status s;
+ auto task = [&] { s = delete_file_impl(path); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::delete_directory(const Path& dir) {
+ auto path = absolute_path(dir);
+ if (bthread_self() == 0) {
+ return delete_directory_impl(path);
+ }
+ Status s;
+ auto task = [&] { s = delete_directory_impl(path); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::batch_delete(const std::vector<Path>& files) {
+ std::vector<Path> abs_files;
+ for (auto& file : files) {
+ abs_files.push_back(absolute_path(file));
+ }
+ if (bthread_self() == 0) {
+ return batch_delete_impl(abs_files);
+ }
+ Status s;
+ auto task = [&] { s = batch_delete_impl(abs_files); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::exists(const Path& path, bool* res) const {
+ auto fs_path = absolute_path(path);
+ if (bthread_self() == 0) {
+ return exists_impl(fs_path, res);
+ }
+ Status s;
+ auto task = [&] { s = exists_impl(fs_path, res); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::file_size(const Path& file, size_t* file_size) const {
+ auto path = absolute_path(file);
+ if (bthread_self() == 0) {
+ return file_size_impl(path, file_size);
+ }
+ Status s;
+ auto task = [&] { s = file_size_impl(path, file_size); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::list(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) {
+ auto path = absolute_path(dir);
+ if (bthread_self() == 0) {
+ return list_impl(path, only_file, files, exists);
+ }
+ Status s;
+ auto task = [&] { s = list_impl(path, only_file, files, exists); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::rename(const Path& orig_name, const Path& new_name) {
+ auto orig_path = absolute_path(orig_name);
+ auto new_path = absolute_path(new_name);
+ if (bthread_self() == 0) {
+ return rename_impl(orig_path, new_path);
+ }
+ Status s;
+ auto task = [&] { s = rename_impl(orig_path, new_path); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status FileSystem::rename_dir(const Path& orig_name, const Path& new_name) {
+ auto orig_path = absolute_path(orig_name);
+ auto new_path = absolute_path(new_name);
+ if (bthread_self() == 0) {
+ return rename_dir_impl(orig_path, new_path);
+ }
+ Status s;
+ auto task = [&] { s = rename_dir_impl(orig_path, new_path); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+} // namespace io
+} // namespace doris
diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h
index 642c842319..05c58b39a7 100644
--- a/be/src/io/fs/file_system.h
+++ b/be/src/io/fs/file_system.h
@@ -21,17 +21,14 @@
#include "common/status.h"
#include "gutil/macros.h"
-#include "io/fs/file_reader.h"
#include "io/fs/file_reader_options.h"
-#include "io/fs/file_writer.h"
+#include "io/fs/file_reader_writer_fwd.h"
#include "io/fs/path.h"
+#include "io/io_common.h"
namespace doris {
namespace io {
-class FileWriter;
-class FileReader;
-
enum class FileSystemType : uint8_t {
LOCAL,
S3,
@@ -39,41 +36,106 @@ enum class FileSystemType : uint8_t {
BROKER,
};
+struct FileInfo {
+ // only file name, no path
+ std::string file_name;
+ size_t file_size;
+ bool is_file;
+};
+
class FileSystem : public std::enable_shared_from_this<FileSystem> {
public:
- virtual ~FileSystem() = default;
-
- DISALLOW_COPY_AND_ASSIGN(FileSystem);
-
- virtual Status create_file(const Path& path, FileWriterPtr* writer) = 0;
-
- virtual Status open_file(const Path& path, const FileReaderOptions& reader_options,
- FileReaderSPtr* reader, IOContext* io_ctx) = 0;
-
- virtual Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) = 0;
-
- virtual Status delete_file(const Path& path) = 0;
-
- // create directory recursively
- virtual Status create_directory(const Path& path) = 0;
-
- // remove all under directory recursively
- virtual Status delete_directory(const Path& path) = 0;
-
- // hard link `src` to `dest`
- // FIXME(cyx): Should we move this method to LocalFileSystem?
- virtual Status link_file(const Path& src, const Path& dest) = 0;
-
- virtual Status exists(const Path& path, bool* res) const = 0;
-
- virtual Status file_size(const Path& path, size_t* file_size) const = 0;
-
- virtual Status list(const Path& path, std::vector<Path>* files) = 0;
+ // The following are public interface.
+ // And derived classes should implement all xxx_impl methods.
+ Status create_file(const Path& file, FileWriterPtr* writer);
+ Status open_file(const Path& file, FileReaderSPtr* reader) {
+ return open_file(file, FileReaderOptions::DEFAULT, reader);
+ }
+ Status open_file(const Path& file, const FileReaderOptions& reader_options,
+ FileReaderSPtr* reader);
+ Status create_directory(const Path& dir);
+ Status delete_file(const Path& file);
+ Status delete_directory(const Path& dir);
+ Status batch_delete(const std::vector<Path>& files);
+ Status exists(const Path& path, bool* res) const;
+ Status file_size(const Path& file, size_t* file_size) const;
+ Status list(const Path& dir, bool only_file, std::vector<FileInfo>* files, bool* exists);
+ Status rename(const Path& orig_name, const Path& new_name);
+ Status rename_dir(const Path& orig_name, const Path& new_name);
+
+ std::shared_ptr<FileSystem> getSPtr() { return shared_from_this(); }
+public:
+ // the root path of this fs.
+ // if not empty, all given Path will be "_root_path/path"
const Path& root_path() const { return _root_path; }
+ // a unique id of this fs.
+ // used for cache or re-use.
+ // can be empty if not used
const std::string& id() const { return _id; }
+ // file system type
FileSystemType type() const { return _type; }
+ virtual ~FileSystem() = default;
+
+ // Each derived class should implement create method to create fs.
+ DISALLOW_COPY_AND_ASSIGN(FileSystem);
+
+protected:
+ /// create file and return a FileWriter
+ virtual Status create_file_impl(const Path& file, FileWriterPtr* writer) = 0;
+
+ /// open file and return a FileReader
+ virtual Status open_file_impl(const Path& file, const FileReaderOptions& reader_options,
+ FileReaderSPtr* reader) = 0;
+
+ /// create directory recursively
+ virtual Status create_directory_impl(const Path& dir) = 0;
+
+ /// delete file.
+ /// return OK if file does not exist
+ /// return ERR if not a regular file
+ virtual Status delete_file_impl(const Path& file) = 0;
+
+ /// delete all files in "files"
+ virtual Status batch_delete_impl(const std::vector<Path>& files) = 0;
+
+ /// remove all under directory recursively
+ /// return OK if dir does not exist
+ /// return ERR if not a dir
+ virtual Status delete_directory_impl(const Path& dir) = 0;
+
+ /// check if path exist
+ /// return OK and res = 1 means exist, res = 0 means does not exist
+ /// return ERR otherwise
+ virtual Status exists_impl(const Path& path, bool* res) const = 0;
+
+ /// return OK and get size of given file, save in "file_size".
+ /// return ERR otherwise
+ virtual Status file_size_impl(const Path& file, size_t* file_size) const = 0;
+
+ /// return OK and list all objects in "dir", save in "files"
+ /// return ERR otherwise
+ /// will not traverse dir recursively.
+ /// if "only_file" is true, will only return regular files, otherwise, return files and subdirs.
+ /// the existence of dir will be saved in "exists"
+ /// if "dir" does not exist, it will return Status::OK, but "exists" will to false
+ virtual Status list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) = 0;
+
+ /// rename file from orig_name to new_name
+ virtual Status rename_impl(const Path& orig_name, const Path& new_name) = 0;
+
+ /// rename dir from orig_name to new_name
+ virtual Status rename_dir_impl(const Path& orig_name, const Path& new_name) = 0;
+
+ virtual Path absolute_path(const Path& path) const {
+ if (path.is_absolute()) {
+ return path;
+ }
+ return _root_path / path;
+ }
+
protected:
FileSystem(Path&& root_path, std::string&& id, FileSystemType type)
: _root_path(std::move(root_path)), _id(std::move(id)), _type(type) {}
diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h
index 9f2859a394..dfd8d2f094 100644
--- a/be/src/io/fs/file_writer.h
+++ b/be/src/io/fs/file_writer.h
@@ -21,16 +21,17 @@
#include "common/status.h"
#include "gutil/macros.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
#include "io/fs/path.h"
#include "util/slice.h"
namespace doris {
namespace io {
-class FileSystem;
class FileWriter {
public:
- FileWriter(Path&& path) : _path(std::move(path)) {}
+ FileWriter(Path&& path, FileSystemSPtr fs) : _path(std::move(path)), _fs(fs) {}
virtual ~FileWriter() = default;
DISALLOW_COPY_AND_ASSIGN(FileWriter);
@@ -41,7 +42,7 @@ public:
// Abnormal close and remove this file.
virtual Status abort() = 0;
- virtual Status append(const Slice& data) = 0;
+ Status append(const Slice& data) { return appendv(&data, 1); }
virtual Status appendv(const Slice* data, size_t data_cnt) = 0;
@@ -51,17 +52,19 @@ public:
// FIXME(cyx): Does not seem to be an appropriate interface for file system?
virtual Status finalize() = 0;
- virtual size_t bytes_appended() const = 0;
+ const Path& path() const { return _path; }
- virtual std::shared_ptr<FileSystem> fs() const = 0;
+ size_t bytes_appended() const { return _bytes_appended; }
- const Path& path() const { return _path; }
+ FileSystemSPtr fs() const { return _fs; }
protected:
Path _path;
+ size_t _bytes_appended = 0;
+ FileSystemSPtr _fs;
+ bool _closed = false;
+ bool _opened = false;
};
-using FileWriterPtr = std::unique_ptr<FileWriter>;
-
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/file_reader_options.cpp b/be/src/io/fs/fs_utils.cpp
similarity index 61%
copy from be/src/io/fs/file_reader_options.cpp
copy to be/src/io/fs/fs_utils.cpp
index 0f8f74dbc2..fbd6a89fc6 100644
--- a/be/src/io/fs/file_reader_options.cpp
+++ b/be/src/io/fs/fs_utils.cpp
@@ -15,21 +15,31 @@
// specific language governing permissions and limitations
// under the License.
-#include "io/fs/file_reader_options.h"
+#include "io/fs/fs_utils.h"
+
+#include <fmt/format.h>
+#include <hdfs/hdfs.h>
+
+#include <sstream>
namespace doris {
namespace io {
-FileCachePolicy cache_type_from_string(const std::string& type) {
- if (type == "sub_file_cache") {
- return FileCachePolicy::SUB_FILE_CACHE;
- } else if (type == "whole_file_cache") {
- return FileCachePolicy::WHOLE_FILE_CACHE;
- } else if (type == "file_block_cache") {
- return FileCachePolicy::FILE_BLOCK_CACHE;
- } else {
- return FileCachePolicy::NO_CACHE;
- }
+std::string errno_to_str() {
+ char buf[1024];
+ return fmt::format("({}), {}", errno, strerror_r(errno, buf, 1024));
+}
+
+std::string errcode_to_str(const std::error_code& ec) {
+ return fmt::format("({}), {}", ec.value(), ec.message());
+}
+
+std::string hdfs_error() {
+ std::stringstream ss;
+ char buf[1024];
+ ss << "(" << errno << "), " << strerror_r(errno, buf, 1024);
+ ss << ", reason: " << hdfsGetLastError();
+ return ss.str();
}
} // namespace io
diff --git a/be/src/io/file_writer.h b/be/src/io/fs/fs_utils.h
similarity index 66%
rename from be/src/io/file_writer.h
rename to be/src/io/fs/fs_utils.h
index 5b08f09830..31ca702c32 100644
--- a/be/src/io/file_writer.h
+++ b/be/src/io/fs/fs_utils.h
@@ -17,23 +17,15 @@
#pragma once
-#include <stdint.h>
-
-#include "common/status.h"
+#include <string>
+#include <system_error>
namespace doris {
+namespace io {
-class FileWriter {
-public:
- virtual ~FileWriter() {}
-
- virtual Status open() = 0;
-
- // Writes up to count bytes from the buffer pointed buf to the file.
- // NOTE: the number of bytes written may be less than count if.
- virtual Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) = 0;
-
- virtual Status close() = 0;
-};
+std::string errno_to_str();
+std::string errcode_to_str(const std::error_code& ec);
+std::string hdfs_error();
-} // end namespace doris
+} // namespace io
+} // namespace doris
diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp
index 738a6a3736..219410ac18 100644
--- a/be/src/io/fs/hdfs_file_reader.cpp
+++ b/be/src/io/fs/hdfs_file_reader.cpp
@@ -54,8 +54,8 @@ Status HdfsFileReader::close() {
return Status::OK();
}
-Status HdfsFileReader::read_at(size_t offset, Slice result, const IOContext& /*io_ctx*/,
- size_t* bytes_read) {
+Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) {
DCHECK(!closed());
if (offset > _file_size) {
return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})",
diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h
index bd25b1d6ba..2b10baa602 100644
--- a/be/src/io/fs/hdfs_file_reader.h
+++ b/be/src/io/fs/hdfs_file_reader.h
@@ -19,6 +19,7 @@
#include "io/fs/file_reader.h"
#include "io/fs/hdfs_file_system.h"
+
namespace doris {
namespace io {
@@ -31,9 +32,6 @@ public:
Status close() override;
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
const Path& path() const override { return _path; }
size_t size() const override { return _file_size; }
@@ -42,6 +40,10 @@ public:
FileSystemSPtr fs() const override { return _fs; }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
Path _path;
size_t _file_size;
diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp
index 8f36600298..e893249eb2 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -19,17 +19,22 @@
#include "gutil/hash/hash.h"
#include "io/cache/block/cached_remote_file_reader.h"
+#include "io/fs/fs_utils.h"
#include "io/fs/hdfs_file_reader.h"
+#include "io/fs/hdfs_file_writer.h"
+#include "io/fs/local_file_system.h"
#include "io/hdfs_builder.h"
#include "service/backend_options.h"
+#include "util/hdfs_util.h"
+#include "util/stack_util.h"
namespace doris {
namespace io {
#ifndef CHECK_HDFS_HANDLE
-#define CHECK_HDFS_HANDLE(handle) \
- if (!handle) { \
- return Status::InternalError("init Hdfs handle error"); \
+#define CHECK_HDFS_HANDLE(handle) \
+ if (!handle) { \
+ return Status::IOError("init Hdfs handle error"); \
}
#endif
@@ -61,9 +66,10 @@ private:
void _clean_oldest();
};
-std::shared_ptr<HdfsFileSystem> HdfsFileSystem::create(const THdfsParams& hdfs_params,
- const std::string& path) {
- return std::shared_ptr<HdfsFileSystem>(new HdfsFileSystem(hdfs_params, path));
+Status HdfsFileSystem::create(const THdfsParams& hdfs_params, const std::string& path,
+ std::shared_ptr<HdfsFileSystem>* fs) {
+ (*fs).reset(new HdfsFileSystem(hdfs_params, path));
+ return (*fs)->connect();
}
HdfsFileSystem::HdfsFileSystem(const THdfsParams& hdfs_params, const std::string& path)
@@ -83,32 +89,25 @@ HdfsFileSystem::~HdfsFileSystem() {
}
}
-Status HdfsFileSystem::connect() {
+Status HdfsFileSystem::connect_impl() {
RETURN_IF_ERROR(HdfsFileSystemCache::instance()->get_connection(_hdfs_params, &_fs_handle));
if (!_fs_handle) {
- return Status::InternalError("failed to init Hdfs handle with, please check hdfs params.");
+ return Status::IOError("failed to init Hdfs handle with, please check hdfs params.");
}
return Status::OK();
}
-Status HdfsFileSystem::create_file(const Path& /*path*/, FileWriterPtr* /*writer*/) {
- // auto handle = get_handle();
- // CHECK_HDFS_HANDLE(handle);
- // auto hdfs_file = hdfsOpenFile(handle->hdfs_fs, path.string().c_str(), O_WRONLY, 0, 0, 0);
- // if (hdfs_file == nullptr) {
- // return Status::InternalError("Failed to create file {}", path.string());
- // }
- // hdfsCloseFile(handle->hdfs_fs, hdfs_file);
- // return Status::OK();
- return Status::NotSupported("Currently not support to create file to HDFS");
+Status HdfsFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer) {
+ *writer = std::make_unique<HdfsFileWriter>(file, getSPtr());
+ return Status::OK();
}
-Status HdfsFileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOContext* /*io_ctx*/) {
+Status HdfsFileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader) {
CHECK_HDFS_HANDLE(_fs_handle);
size_t file_len = 0;
- RETURN_IF_ERROR(file_size(path, &file_len));
+ RETURN_IF_ERROR(file_size_impl(file, &file_len));
- Path real_path = _covert_path(path);
+ Path real_path = convert_path(file, _namenode);
auto hdfs_file =
hdfsOpenFile(_fs_handle->hdfs_fs, real_path.string().c_str(), O_RDONLY, 0, 0, 0);
if (hdfs_file == nullptr) {
@@ -118,112 +117,249 @@ Status HdfsFileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOCon
_fs_handle->dec_ref();
// retry
RETURN_IF_ERROR(connect());
- hdfs_file = hdfsOpenFile(_fs_handle->hdfs_fs, path.string().c_str(), O_RDONLY, 0, 0, 0);
+ hdfs_file = hdfsOpenFile(_fs_handle->hdfs_fs, file.string().c_str(), O_RDONLY, 0, 0, 0);
if (hdfs_file == nullptr) {
- return Status::InternalError(
- "open file failed. (BE: {}) namenode:{}, path:{}, err: {}",
- BackendOptions::get_localhost(), _namenode, path.string(),
- hdfsGetLastError());
+ return Status::IOError("failed to open {}: {}", file.native(), hdfs_error());
}
} else {
- return Status::InternalError("open file failed. (BE: {}) namenode:{}, path:{}, err: {}",
- BackendOptions::get_localhost(), _namenode, path.string(),
- hdfsGetLastError());
+ return Status::IOError("failed to open {} from cache: {}", file.native(), hdfs_error());
}
}
*reader = std::make_shared<HdfsFileReader>(
- path, file_len, _namenode, hdfs_file,
+ file, file_len, _namenode, hdfs_file,
std::static_pointer_cast<HdfsFileSystem>(shared_from_this()));
return Status::OK();
}
-Status HdfsFileSystem::delete_file(const Path& path) {
+Status HdfsFileSystem::create_directory_impl(const Path& dir) {
CHECK_HDFS_HANDLE(_fs_handle);
- Path real_path = _covert_path(path);
- // The recursive argument `is_recursive` is irrelevant if path is a file.
- int is_recursive = 0;
- int res = hdfsDelete(_fs_handle->hdfs_fs, real_path.string().c_str(), is_recursive);
+ Path real_path = convert_path(dir, _namenode);
+ int res = hdfsCreateDirectory(_fs_handle->hdfs_fs, real_path.string().c_str());
if (res == -1) {
- return Status::InternalError("Failed to delete file {}", path.string());
+ return Status::IOError("failed to create directory {}: {}", dir.native(), hdfs_error());
}
return Status::OK();
}
-Status HdfsFileSystem::create_directory(const Path& path) {
- CHECK_HDFS_HANDLE(_fs_handle);
- Path real_path = _covert_path(path);
- int res = hdfsCreateDirectory(_fs_handle->hdfs_fs, real_path.string().c_str());
- if (res == -1) {
- return Status::InternalError("Failed to create directory {}", path.string());
+Status HdfsFileSystem::delete_file_impl(const Path& file) {
+ return delete_internal(file, 0);
+}
+
+Status HdfsFileSystem::delete_directory_impl(const Path& dir) {
+ return delete_internal(dir, 1);
+}
+
+Status HdfsFileSystem::batch_delete_impl(const std::vector<Path>& files) {
+ for (auto& file : files) {
+ RETURN_IF_ERROR(delete_file_impl(file));
}
return Status::OK();
}
-Status HdfsFileSystem::delete_directory(const Path& path) {
+Status HdfsFileSystem::delete_internal(const Path& path, int is_recursive) {
+ bool exists = true;
+ RETURN_IF_ERROR(exists_impl(path, &exists));
+ if (!exists) {
+ return Status::OK();
+ }
CHECK_HDFS_HANDLE(_fs_handle);
- Path real_path = _covert_path(path);
- // delete in recursive mode
- int is_recursive = 1;
+ Path real_path = convert_path(path, _namenode);
int res = hdfsDelete(_fs_handle->hdfs_fs, real_path.string().c_str(), is_recursive);
if (res == -1) {
- return Status::InternalError("Failed to delete directory {}", path.string());
+ return Status::IOError("failed to delete directory {}: {}", path.native(), hdfs_error());
}
return Status::OK();
}
-Status HdfsFileSystem::exists(const Path& path, bool* res) const {
+Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const {
CHECK_HDFS_HANDLE(_fs_handle);
- Path real_path = _covert_path(path);
+ Path real_path = convert_path(path, _namenode);
int is_exists = hdfsExists(_fs_handle->hdfs_fs, real_path.string().c_str());
- if (is_exists == 0) {
- *res = true;
- } else {
- *res = false;
- }
+ *res = (is_exists == 0);
return Status::OK();
}
-Status HdfsFileSystem::file_size(const Path& path, size_t* file_size) const {
+Status HdfsFileSystem::file_size_impl(const Path& path, size_t* file_size) const {
CHECK_HDFS_HANDLE(_fs_handle);
- Path real_path = _covert_path(path);
+ Path real_path = convert_path(path, _namenode);
hdfsFileInfo* file_info = hdfsGetPathInfo(_fs_handle->hdfs_fs, real_path.string().c_str());
if (file_info == nullptr) {
- return Status::InternalError("Failed to get file size of {}", path.string());
+ return Status::IOError("failed to get file size of {}: {}", path.native(), hdfs_error());
}
*file_size = file_info->mSize;
hdfsFreeFileInfo(file_info, 1);
return Status::OK();
}
-Status HdfsFileSystem::list(const Path& path, std::vector<Path>* files) {
+Status HdfsFileSystem::list_impl(const Path& path, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) {
+ RETURN_IF_ERROR(exists_impl(path, exists));
+ if (!(*exists)) {
+ return Status::OK();
+ }
+
CHECK_HDFS_HANDLE(_fs_handle);
- Path real_path = _covert_path(path);
+ Path real_path = convert_path(path, _namenode);
int numEntries = 0;
- hdfsFileInfo* file_info =
- hdfsListDirectory(_fs_handle->hdfs_fs, real_path.string().c_str(), &numEntries);
- if (file_info == nullptr) {
- return Status::InternalError("Failed to list files/directors of {}", path.string());
+ hdfsFileInfo* hdfs_file_info =
+ hdfsListDirectory(_fs_handle->hdfs_fs, real_path.c_str(), &numEntries);
+ if (hdfs_file_info == nullptr) {
+ return Status::IOError("failed to list files/directors {}: {}", path.native(),
+ hdfs_error());
}
for (int idx = 0; idx < numEntries; ++idx) {
- files->emplace_back(file_info[idx].mName);
+ auto& file = hdfs_file_info[idx];
+ if (only_file && file.mKind == kObjectKindDirectory) {
+ continue;
+ }
+ FileInfo file_info;
+ file_info.file_name = file.mName;
+ file_info.file_size = file.mSize;
+ file_info.is_file = (file.mKind != kObjectKindDirectory);
+ files->emplace_back(std::move(file_info));
}
- hdfsFreeFileInfo(file_info, numEntries);
+ hdfsFreeFileInfo(hdfs_file_info, numEntries);
return Status::OK();
}
-HdfsFileSystemHandle* HdfsFileSystem::get_handle() {
- return _fs_handle;
+Status HdfsFileSystem::rename_impl(const Path& orig_name, const Path& new_name) {
+ Path normal_orig_name = convert_path(orig_name, _namenode);
+ Path normal_new_name = convert_path(new_name, _namenode);
+ int ret = hdfsRename(_fs_handle->hdfs_fs, normal_orig_name.c_str(), normal_new_name.c_str());
+ if (ret == 0) {
+ LOG(INFO) << "finished to rename file. orig: " << normal_orig_name
+ << ", new: " << normal_new_name;
+ return Status::OK();
+ } else {
+ return Status::IOError("fail to rename from {} to {}: {}", normal_orig_name.native(),
+ normal_new_name.native(), hdfs_error());
+ }
+ return Status::OK();
}
-Path HdfsFileSystem::_covert_path(const Path& path) const {
- // if the format of path is hdfs://ip:port/path, replace it to /path.
- // path like hdfs://ip:port/path can't be used by libhdfs3.
- Path real_path(path);
- if (path.string().find(_namenode) != std::string::npos) {
- std::string real_path_str = path.string().substr(_namenode.size());
- real_path = real_path_str;
+Status HdfsFileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) {
+ return rename_impl(orig_name, new_name);
+}
+
+Status HdfsFileSystem::upload_impl(const Path& local_file, const Path& remote_file) {
+ // 1. open local file for read
+ FileSystemSPtr local_fs = global_local_filesystem();
+ FileReaderSPtr local_reader = nullptr;
+ RETURN_IF_ERROR(local_fs->open_file(local_file, &local_reader));
+ size_t file_len = local_reader->size();
+ if (file_len == -1) {
+ return Status::IOError("failed to get size of file: {}", local_file.string());
+ }
+
+ // 2. open remote file for write
+ FileWriterPtr hdfs_writer = nullptr;
+ RETURN_IF_ERROR(create_file_impl(remote_file, &hdfs_writer));
+
+ constexpr size_t buf_sz = 1024 * 1024;
+ char read_buf[buf_sz];
+ size_t left_len = file_len;
+ size_t read_offset = 0;
+ size_t bytes_read = 0;
+ while (left_len > 0) {
+ size_t read_len = left_len > buf_sz ? buf_sz : left_len;
+ RETURN_IF_ERROR(local_reader->read_at(read_offset, {read_buf, read_len}, &bytes_read));
+ RETURN_IF_ERROR(hdfs_writer->append({read_buf, read_len}));
+
+ read_offset += read_len;
+ left_len -= read_len;
}
- return real_path;
+
+ LOG(INFO) << "finished to write file: " << local_file << ", length: " << file_len;
+ return Status::OK();
+}
+
+Status HdfsFileSystem::batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) {
+ DCHECK(local_files.size() == remote_files.size());
+ for (int i = 0; i < local_files.size(); ++i) {
+ RETURN_IF_ERROR(upload_impl(local_files[i], remote_files[i]));
+ }
+ return Status::OK();
+}
+
+Status HdfsFileSystem::direct_upload_impl(const Path& remote_file, const std::string& content) {
+ FileWriterPtr hdfs_writer = nullptr;
+ RETURN_IF_ERROR(create_file(remote_file, &hdfs_writer));
+ RETURN_IF_ERROR(hdfs_writer->append({content}));
+ return Status::OK();
+}
+
+Status HdfsFileSystem::upload_with_checksum_impl(const Path& local, const Path& remote_file,
+ const std::string& checksum) {
+ std::string temp = remote_file.string() + ".part";
+ std::string final_file = remote_file.string() + "." + checksum;
+ RETURN_IF_ERROR(upload_impl(local, temp));
+ return rename_impl(temp, final_file);
+}
+
+Status HdfsFileSystem::download_impl(const Path& remote_file, const Path& local_file) {
+ // 1. open remote file for read
+ FileReaderSPtr hdfs_reader = nullptr;
+ RETURN_IF_ERROR(open_file_internal(remote_file, &hdfs_reader));
+
+ // 2. remove the existing local file if exist
+ if (std::filesystem::remove(local_file)) {
+ LOG(INFO) << "remove the previously exist local file: " << local_file;
+ }
+
+ // 3. open local file for write
+ FileSystemSPtr local_fs = global_local_filesystem();
+ FileWriterPtr local_writer = nullptr;
+ RETURN_IF_ERROR(local_fs->create_file(local_file, &local_writer));
+
+ // 4. read remote and write to local
+ LOG(INFO) << "read remote file: " << remote_file << " to local: " << local_file;
+ constexpr size_t buf_sz = 1024 * 1024;
+ std::unique_ptr<char[]> read_buf(new char[buf_sz]);
+ size_t write_offset = 0;
+ size_t cur_offset = 0;
+ while (true) {
+ size_t read_len = 0;
+ Slice file_slice(read_buf.get(), buf_sz);
+ RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, &read_len));
+ cur_offset += read_len;
+ if (read_len == 0) {
+ break;
+ }
+
+ RETURN_IF_ERROR(local_writer->write_at(write_offset, {read_buf.get(), read_len}));
+ write_offset += read_len;
+ }
+
+ return Status::OK();
+}
+
+Status HdfsFileSystem::direct_download_impl(const Path& remote_file, std::string* content) {
+ // 1. open remote file for read
+ FileReaderSPtr hdfs_reader = nullptr;
+ RETURN_IF_ERROR(open_file_internal(remote_file, &hdfs_reader));
+
+ constexpr size_t buf_sz = 1024 * 1024;
+ std::unique_ptr<char[]> read_buf(new char[buf_sz]);
+ size_t write_offset = 0;
+ size_t cur_offset = 0;
+ while (true) {
+ size_t read_len = 0;
+ Slice file_slice(read_buf.get(), buf_sz);
+ RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, &read_len));
+ cur_offset += read_len;
+ if (read_len == 0) {
+ break;
+ }
+
+ content->insert(write_offset, read_buf.get(), read_len);
+ write_offset += read_len;
+ }
+ return Status::OK();
+}
+
+HdfsFileSystemHandle* HdfsFileSystem::get_handle() {
+ return _fs_handle;
}
// ************* HdfsFileSystemCache ******************
@@ -234,7 +370,8 @@ Status HdfsFileSystemCache::_create_fs(const THdfsParams& hdfs_params, hdfsFS* f
RETURN_IF_ERROR(createHDFSBuilder(hdfs_params, &builder));
hdfsFS hdfs_fs = hdfsBuilderConnect(builder.get());
if (hdfs_fs == nullptr) {
- return Status::InternalError("connect to hdfs failed. error: {}", hdfsGetLastError());
+ return Status::IOError("faield to connect to hdfs {}: {}", hdfs_params.fs_name,
+ hdfs_error());
}
*fs = hdfs_fs;
return Status::OK();
diff --git a/be/src/io/fs/hdfs_file_system.h b/be/src/io/fs/hdfs_file_system.h
index 9e5edf6752..41ec7b7243 100644
--- a/be/src/io/fs/hdfs_file_system.h
+++ b/be/src/io/fs/hdfs_file_system.h
@@ -81,49 +81,43 @@ private:
class HdfsFileSystem final : public RemoteFileSystem {
public:
- static std::shared_ptr<HdfsFileSystem> create(const THdfsParams& hdfs_params,
- const std::string& path);
+ static Status create(const THdfsParams& hdfs_params, const std::string& path,
+ std::shared_ptr<HdfsFileSystem>* fs);
~HdfsFileSystem() override;
- Status create_file(const Path& path, FileWriterPtr* writer) override;
-
- Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override;
-
- Status delete_file(const Path& path) override;
-
- Status create_directory(const Path& path) override;
-
- // Delete all files under path.
- Status delete_directory(const Path& path) override;
-
- Status link_file(const Path& /*src*/, const Path& /*dest*/) override {
- return Status::NotSupported("Not supported");
- }
-
- Status exists(const Path& path, bool* res) const override;
-
- Status file_size(const Path& path, size_t* file_size) const override;
-
- Status list(const Path& path, std::vector<Path>* files) override;
-
- Status upload(const Path& /*local_path*/, const Path& /*dest_path*/) override {
- return Status::NotSupported("Currently not support to upload file to HDFS");
- }
-
- Status batch_upload(const std::vector<Path>& /*local_paths*/,
- const std::vector<Path>& /*dest_paths*/) override {
- return Status::NotSupported("Currently not support to batch upload file to HDFS");
- }
+ HdfsFileSystemHandle* get_handle();
- Status connect() override;
+protected:
+ Status connect_impl() override;
+ Status create_file_impl(const Path& file, FileWriterPtr* writer) override;
+ Status open_file_internal(const Path& file, FileReaderSPtr* reader) override;
+ Status create_directory_impl(const Path& dir) override;
+ Status delete_file_impl(const Path& file) override;
+ Status delete_directory_impl(const Path& dir) override;
+ Status batch_delete_impl(const std::vector<Path>& files) override;
+ Status exists_impl(const Path& path, bool* res) const override;
+ Status file_size_impl(const Path& file, size_t* file_size) const override;
+ Status list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) override;
+ Status rename_impl(const Path& orig_name, const Path& new_name) override;
+ Status rename_dir_impl(const Path& orig_name, const Path& new_name) override;
+
+ Status upload_impl(const Path& local_file, const Path& remote_file) override;
+ Status batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) override;
+ Status direct_upload_impl(const Path& remote_file, const std::string& content) override;
+ Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file,
+ const std::string& checksum) override;
+ Status download_impl(const Path& remote_file, const Path& local_file) override;
+ Status direct_download_impl(const Path& remote_file, std::string* content) override;
- HdfsFileSystemHandle* get_handle();
+private:
+ Status delete_internal(const Path& path, int is_recursive);
private:
+ friend class HdfsFileWriter;
HdfsFileSystem(const THdfsParams& hdfs_params, const std::string& path);
-
- Path _covert_path(const Path& path) const;
const THdfsParams& _hdfs_params;
std::string _namenode;
// do not use std::shared_ptr or std::unique_ptr
diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp
new file mode 100644
index 0000000000..71b3de77e5
--- /dev/null
+++ b/be/src/io/fs/hdfs_file_writer.cpp
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/hdfs_file_writer.h"
+
+#include <filesystem>
+
+#include "common/logging.h"
+#include "io/fs/hdfs_file_system.h"
+#include "service/backend_options.h"
+#include "util/hdfs_util.h"
+
+namespace doris {
+namespace io {
+
+HdfsFileWriter::HdfsFileWriter(Path file, FileSystemSPtr fs) : FileWriter(std::move(file), fs) {
+ _hdfs_fs = (HdfsFileSystem*)_fs.get();
+}
+
+HdfsFileWriter::~HdfsFileWriter() {
+ if (_opened) {
+ close();
+ }
+ CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed;
+}
+
+Status HdfsFileWriter::close() {
+ if (_closed) {
+ return Status::OK();
+ }
+ _closed = true;
+ if (_hdfs_file == nullptr) {
+ return Status::OK();
+ }
+ int result = hdfsFlush(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file);
+ if (result == -1) {
+ std::stringstream ss;
+ ss << "failed to flush hdfs file. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _hdfs_fs->_namenode << " path:" << _path
+ << ", err: " << hdfsGetLastError();
+ LOG(WARNING) << ss.str();
+ return Status::InternalError(ss.str());
+ }
+ hdfsCloseFile(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file);
+ _hdfs_file = nullptr;
+ return Status::OK();
+}
+
+Status HdfsFileWriter::abort() {
+ // TODO: should delete remote file
+ return Status::OK();
+}
+
+Status HdfsFileWriter::appendv(const Slice* data, size_t data_cnt) {
+ DCHECK(!_closed);
+ if (!_opened) {
+ RETURN_IF_ERROR(_open());
+ _opened = true;
+ }
+
+ for (size_t i = 0; i < data_cnt; i++) {
+ const Slice& result = data[i];
+ size_t left_bytes = result.size;
+ const char* p = result.data;
+ while (left_bytes > 0) {
+ int64_t written_bytes =
+ hdfsWrite(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file, p, left_bytes);
+ if (written_bytes < 0) {
+ return Status::InternalError("write hdfs failed. namenode: {}, path: {}, error: {}",
+ _hdfs_fs->_namenode, _path.native(),
+ hdfsGetLastError());
+ }
+ left_bytes -= written_bytes;
+ p += written_bytes;
+ _bytes_appended += written_bytes;
+ }
+ }
+ return Status::OK();
+}
+
+// Call this method when there is no more data to write.
+// FIXME(cyx): Does not seem to be an appropriate interface for file system?
+Status HdfsFileWriter::finalize() {
+ DCHECK(!_closed);
+ if (_opened) {
+ RETURN_IF_ERROR(close());
+ }
+ return Status::OK();
+}
+
+Status HdfsFileWriter::_open() {
+ _path = convert_path(_path, _hdfs_fs->_namenode);
+ std::string hdfs_dir = _path.parent_path().string();
+ int exists = hdfsExists(_hdfs_fs->_fs_handle->hdfs_fs, hdfs_dir.c_str());
+ if (exists != 0) {
+ VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir;
+ int ret = hdfsCreateDirectory(_hdfs_fs->_fs_handle->hdfs_fs, hdfs_dir.c_str());
+ if (ret != 0) {
+ std::stringstream ss;
+ ss << "create dir failed. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode: " << _hdfs_fs->_namenode << " path: " << hdfs_dir
+ << ", err: " << hdfsGetLastError();
+ LOG(WARNING) << ss.str();
+ return Status::InternalError(ss.str());
+ }
+ }
+ // open file
+ _hdfs_file = hdfsOpenFile(_hdfs_fs->_fs_handle->hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0);
+ if (_hdfs_file == nullptr) {
+ std::stringstream ss;
+ ss << "open file failed. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode:" << _hdfs_fs->_namenode << " path:" << _path
+ << ", err: " << hdfsGetLastError();
+ LOG(WARNING) << ss.str();
+ return Status::InternalError(ss.str());
+ }
+ VLOG_NOTICE << "open file. namenode:" << _hdfs_fs->_namenode << ", path:" << _path;
+ return Status::OK();
+}
+
+} // end namespace io
+} // end namespace doris
diff --git a/be/src/io/hdfs_writer.h b/be/src/io/fs/hdfs_file_writer.h
similarity index 60%
rename from be/src/io/hdfs_writer.h
rename to be/src/io/fs/hdfs_file_writer.h
index f5ef32a238..92130a93c2 100644
--- a/be/src/io/hdfs_writer.h
+++ b/be/src/io/fs/hdfs_file_writer.h
@@ -21,33 +21,35 @@
#include <string>
#include "gen_cpp/PlanNodes_types.h"
-#include "io/file_writer.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/path.h"
#include "io/hdfs_builder.h"
namespace doris {
-class HDFSWriter : public FileWriter {
-public:
- HDFSWriter(const std::map<std::string, std::string>& properties, const std::string& path);
- ~HDFSWriter();
- Status open() override;
+namespace io {
- // Writes up to count bytes from the buffer pointed buf to the file.
- // NOTE: the number of bytes written may be less than count if.
- Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override;
+class HdfsFileSystem;
+class HdfsFileWriter : public FileWriter {
+public:
+ HdfsFileWriter(Path file, FileSystemSPtr fs);
+ ~HdfsFileWriter();
Status close() override;
+ Status abort() override;
+ Status appendv(const Slice* data, size_t data_cnt) override;
+ Status finalize() override;
+ Status write_at(size_t offset, const Slice& data) override {
+ return Status::NotSupported("not support");
+ }
private:
- Status _connect();
- void _parse_properties(const std::map<std::string, std::string>& prop);
+ Status _open();
private:
- std::map<std::string, std::string> _properties;
- std::string _namenode = "";
- std::string _path = "";
- hdfsFS _hdfs_fs = nullptr;
hdfsFile _hdfs_file = nullptr;
- bool _closed = false;
+ // A convenient pointer to _fs
+ HdfsFileSystem* _hdfs_fs;
};
+} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/local_file_reader.cpp b/be/src/io/fs/local_file_reader.cpp
index afff78bea3..7139950a09 100644
--- a/be/src/io/fs/local_file_reader.cpp
+++ b/be/src/io/fs/local_file_reader.cpp
@@ -54,19 +54,9 @@ Status LocalFileReader::close() {
}
return Status::OK();
}
-Status LocalFileReader::read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
- if (bthread_self() == 0) {
- return read_at_impl(offset, result, io_ctx, bytes_read);
- }
- Status s;
- auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-Status LocalFileReader::read_at_impl(size_t offset, Slice result, const IOContext& /*io_ctx*/,
- size_t* bytes_read) {
+Status LocalFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) {
DCHECK(!closed());
if (offset > _file_size) {
return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})",
diff --git a/be/src/io/fs/local_file_reader.h b/be/src/io/fs/local_file_reader.h
index e2e4903cc7..1f74e7b4be 100644
--- a/be/src/io/fs/local_file_reader.h
+++ b/be/src/io/fs/local_file_reader.h
@@ -34,11 +34,6 @@ public:
Status close() override;
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
- Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read);
-
const Path& path() const override { return _path; }
size_t size() const override { return _file_size; }
@@ -47,6 +42,10 @@ public:
FileSystemSPtr fs() const override { return _fs; }
+private:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
int _fd = -1; // owned
Path _path;
diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index 1a12b7375a..b88f8fbc47 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -18,6 +18,7 @@
#include "io/fs/local_file_system.h"
#include "io/fs/file_system.h"
+#include "io/fs/fs_utils.h"
#include "io/fs/local_file_reader.h"
#include "io/fs/local_file_writer.h"
#include "util/async_io.h"
@@ -34,218 +35,155 @@ LocalFileSystem::LocalFileSystem(Path&& root_path, std::string&& id)
LocalFileSystem::~LocalFileSystem() = default;
-Path LocalFileSystem::absolute_path(const Path& path) const {
- if (path.is_absolute()) {
- return path;
- }
- return _root_path / path;
-}
-
-Status LocalFileSystem::create_file(const Path& path, FileWriterPtr* writer) {
- if (bthread_self() == 0) {
- return create_file_impl(path, writer);
- }
- Status s;
- auto task = [&] { s = create_file_impl(path, writer); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-
-Status LocalFileSystem::create_file_impl(const Path& path, FileWriterPtr* writer) {
- auto fs_path = absolute_path(path);
- int fd = ::open(fs_path.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
+Status LocalFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer) {
+ int fd = ::open(file.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
if (-1 == fd) {
- return Status::IOError("cannot open {}: {}", fs_path.native(), std::strerror(errno));
+ return Status::IOError("failed to open {}: {}", file.native(), errno_to_str());
}
*writer = std::make_unique<LocalFileWriter>(
- std::move(fs_path), fd, std::static_pointer_cast<LocalFileSystem>(shared_from_this()));
+ std::move(file), fd, std::static_pointer_cast<LocalFileSystem>(shared_from_this()));
return Status::OK();
}
-Status LocalFileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) {
- if (bthread_self() == 0) {
- return open_file_impl(path, reader, io_ctx);
- }
- Status s;
- auto task = [&] { s = open_file_impl(path, reader, io_ctx); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-
-Status LocalFileSystem::open_file_impl(const Path& path, FileReaderSPtr* reader,
- IOContext* /*io_ctx*/) {
- auto fs_path = absolute_path(path);
+Status LocalFileSystem::open_file_impl(const Path& file,
+ const FileReaderOptions& /*reader_options*/,
+ FileReaderSPtr* reader) {
size_t fsize = 0;
- RETURN_IF_ERROR(file_size(fs_path, &fsize));
+ RETURN_IF_ERROR(file_size_impl(file, &fsize));
int fd = -1;
- RETRY_ON_EINTR(fd, open(fs_path.c_str(), O_RDONLY));
+ RETRY_ON_EINTR(fd, open(file.c_str(), O_RDONLY));
if (fd < 0) {
- return Status::IOError("cannot open {}: {}", fs_path.native(), std::strerror(errno));
+ return Status::IOError("failed to open {}: {}", file.native(), errno_to_str());
}
*reader = std::make_shared<LocalFileReader>(
- std::move(fs_path), fsize, fd,
+ std::move(file), fsize, fd,
std::static_pointer_cast<LocalFileSystem>(shared_from_this()));
return Status::OK();
}
-Status LocalFileSystem::delete_file(const Path& path) {
- if (bthread_self() == 0) {
- return delete_file_impl(path);
- }
- Status s;
- auto task = [&] { s = delete_file_impl(path); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-
-Status LocalFileSystem::delete_file_impl(const Path& path) {
- auto fs_path = absolute_path(path);
- if (!std::filesystem::exists(fs_path)) {
- return Status::OK();
- }
- if (!std::filesystem::is_regular_file(fs_path)) {
- return Status::IOError("{} is not a file", fs_path.native());
+Status LocalFileSystem::create_directory_impl(const Path& dir) {
+ if (std::filesystem::exists(dir)) {
+ return Status::IOError("failed to create {}, already exists", dir.native());
}
std::error_code ec;
- std::filesystem::remove(fs_path, ec);
+ std::filesystem::create_directories(dir, ec);
if (ec) {
- return Status::IOError("cannot delete {}: {}", fs_path.native(), std::strerror(ec.value()));
+ return Status::IOError("failed to create {}: {}", dir.native(), errcode_to_str(ec));
}
return Status::OK();
}
-Status LocalFileSystem::create_directory(const Path& path) {
- if (bthread_self() == 0) {
- return create_directory_impl(path);
+Status LocalFileSystem::delete_file_impl(const Path& file) {
+ if (!std::filesystem::exists(file)) {
+ return Status::OK();
}
- Status s;
- auto task = [&] { s = create_directory_impl(path); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-
-Status LocalFileSystem::create_directory_impl(const Path& path) {
- auto fs_path = absolute_path(path);
- if (std::filesystem::exists(fs_path)) {
- return Status::IOError("{} exists", fs_path.native());
+ if (!std::filesystem::is_regular_file(file)) {
+ return Status::IOError("failed to delete {}, not a file", file.native());
}
std::error_code ec;
- std::filesystem::create_directories(fs_path, ec);
+ std::filesystem::remove(file, ec);
if (ec) {
- return Status::IOError("cannot create {}: {}", fs_path.native(), std::strerror(ec.value()));
+ return Status::IOError("failed to delete {}: {}", file.native(), errcode_to_str(ec));
}
return Status::OK();
}
-Status LocalFileSystem::delete_directory(const Path& path) {
- if (bthread_self() == 0) {
- return delete_directory_impl(path);
- }
- Status s;
- auto task = [&] { s = delete_directory_impl(path); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-
-Status LocalFileSystem::delete_directory_impl(const Path& path) {
- auto fs_path = absolute_path(path);
- if (!std::filesystem::exists(fs_path)) {
+Status LocalFileSystem::delete_directory_impl(const Path& dir) {
+ if (!std::filesystem::exists(dir)) {
return Status::OK();
}
- if (!std::filesystem::is_directory(fs_path)) {
- return Status::IOError("{} is not a directory", fs_path.native());
+ if (!std::filesystem::is_directory(dir)) {
+ return Status::IOError("failed to delete {}, not a directory", dir.native());
}
std::error_code ec;
- std::filesystem::remove_all(fs_path, ec);
+ std::filesystem::remove_all(dir, ec);
if (ec) {
- return Status::IOError("cannot delete {}: {}", fs_path.native(), std::strerror(ec.value()));
+ return Status::IOError("failed to delete {}: {}", dir.native(), errcode_to_str(ec));
}
return Status::OK();
}
-Status LocalFileSystem::link_file(const Path& src, const Path& dest) {
- if (bthread_self() == 0) {
- return link_file_impl(src, dest);
+Status LocalFileSystem::batch_delete_impl(const std::vector<Path>& files) {
+ for (auto& file : files) {
+ RETURN_IF_ERROR(delete_file_impl(file));
}
- Status s;
- auto task = [&] { s = link_file_impl(src, dest); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
+ return Status::OK();
}
-Status LocalFileSystem::link_file_impl(const Path& src, const Path& dest) {
- if (::link(src.c_str(), dest.c_str()) != 0) {
- return Status::IOError("fail to create hard link: {}. from {} to {}", std::strerror(errno),
- src.native(), dest.native());
- }
+Status LocalFileSystem::exists_impl(const Path& path, bool* res) const {
+ *res = std::filesystem::exists(path);
return Status::OK();
}
-Status LocalFileSystem::exists(const Path& path, bool* res) const {
- if (bthread_self() == 0) {
- return exists_impl(path, res);
+Status LocalFileSystem::file_size_impl(const Path& file, size_t* file_size) const {
+ std::error_code ec;
+ *file_size = std::filesystem::file_size(file, ec);
+ if (ec) {
+ return Status::IOError("failed to get file size {}: {}", file.native(), errcode_to_str(ec));
}
- Status s;
- auto task = [&] { s = exists_impl(path, res); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
-}
-
-Status LocalFileSystem::exists_impl(const Path& path, bool* res) const {
- auto fs_path = absolute_path(path);
- *res = std::filesystem::exists(fs_path);
return Status::OK();
}
-Status LocalFileSystem::file_size(const Path& path, size_t* file_size) const {
- if (bthread_self() == 0) {
- return file_size_impl(path, file_size);
+Status LocalFileSystem::list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) {
+ if (!std::filesystem::exists(dir)) {
+ *exists = false;
+ return Status::OK();
}
-
- Status s;
- auto task = [&] { s = file_size_impl(path, file_size); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
- return s;
+ std::error_code ec;
+ for (const auto& entry : std::filesystem::directory_iterator(dir, ec)) {
+ if (only_file && !entry.is_regular_file()) {
+ continue;
+ }
+ FileInfo file_info;
+ file_info.file_name = entry.path().filename();
+ file_info.file_size = entry.file_size();
+ file_info.is_file = entry.is_regular_file();
+ files->push_back(std::move(file_info));
+ }
+ if (ec) {
+ return Status::IOError("failed to list {}: {}", dir.native(), errcode_to_str(ec));
+ }
+ return Status::OK();
}
-Status LocalFileSystem::file_size_impl(const Path& path, size_t* file_size) const {
- auto fs_path = absolute_path(path);
+Status LocalFileSystem::rename_impl(const Path& orig_name, const Path& new_name) {
std::error_code ec;
- *file_size = std::filesystem::file_size(fs_path, ec);
+ std::filesystem::rename(orig_name, new_name, ec);
if (ec) {
- return Status::IOError("cannot get file size {}: {}", fs_path.native(),
- std::strerror(ec.value()));
+ return Status::IOError("failed to rename {} to {}: {}", orig_name.native(),
+ new_name.native(), errcode_to_str(ec));
}
return Status::OK();
}
-Status LocalFileSystem::list(const Path& path, std::vector<Path>* files) {
+Status LocalFileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) {
+ return rename_impl(orig_name, new_name);
+}
+
+Status LocalFileSystem::link_file(const Path& src, const Path& dest) {
+ auto src_file = absolute_path(src);
+ auto dest_file = absolute_path(dest);
if (bthread_self() == 0) {
- return list_impl(path, files);
+ return link_file_impl(src_file, dest_file);
}
-
Status s;
- auto task = [&] { s = list_impl(path, files); };
- AsyncIO::run_task(task, io::FileSystemType::LOCAL);
+ auto task = [&] { s = link_file_impl(src_file, dest_file); };
+ AsyncIO::run_task(task, _type);
return s;
}
-Status LocalFileSystem::list_impl(const Path& path, std::vector<Path>* files) {
- files->clear();
- auto fs_path = absolute_path(path);
- std::error_code ec;
- for (const auto& entry : std::filesystem::directory_iterator(fs_path, ec)) {
- files->push_back(entry.path().filename());
- }
- if (ec) {
- return Status::IOError("cannot list {}: {}", fs_path.native(), std::strerror(ec.value()));
+Status LocalFileSystem::link_file_impl(const Path& src, const Path& dest) {
+ if (::link(src.c_str(), dest.c_str()) != 0) {
+ return Status::IOError("failed to create hard link from {} to {}: {}", src.native(),
+ dest.native(), errno_to_str());
}
return Status::OK();
}
-static FileSystemSPtr local_fs = io::LocalFileSystem::create("");
+static std::shared_ptr<LocalFileSystem> local_fs = io::LocalFileSystem::create("");
-const FileSystemSPtr& global_local_filesystem() {
+const std::shared_ptr<LocalFileSystem>& global_local_filesystem() {
return local_fs;
}
diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h
index 2251904616..44202c8995 100644
--- a/be/src/io/fs/local_file_system.h
+++ b/be/src/io/fs/local_file_system.h
@@ -25,57 +25,32 @@ namespace io {
class LocalFileSystem final : public FileSystem {
public:
static std::shared_ptr<LocalFileSystem> create(Path path, std::string id = "");
-
~LocalFileSystem() override;
- Status create_file(const Path& path, FileWriterPtr* writer) override;
-
- Status create_file_impl(const Path& path, FileWriterPtr* writer);
-
- Status open_file(const Path& path, const FileReaderOptions& reader_options,
- FileReaderSPtr* reader, IOContext* io_ctx) override {
- return open_file(path, reader, io_ctx);
- }
-
- Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override;
-
- Status open_file_impl(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx);
-
- Status delete_file(const Path& path) override;
-
- Status delete_file_impl(const Path& path);
-
- Status create_directory(const Path& path) override;
-
- Status create_directory_impl(const Path& path);
-
- Status delete_directory(const Path& path) override;
-
- Status delete_directory_impl(const Path& path);
-
- Status link_file(const Path& src, const Path& dest) override;
-
+ /// hard link dest file to src file
+ Status link_file(const Path& src, const Path& dest);
+
+protected:
+ Status create_file_impl(const Path& file, FileWriterPtr* writer) override;
+ Status open_file_impl(const Path& file, const FileReaderOptions& reader_options,
+ FileReaderSPtr* reader) override;
+ Status create_directory_impl(const Path& dir) override;
+ Status delete_file_impl(const Path& file) override;
+ Status delete_directory_impl(const Path& dir) override;
+ Status batch_delete_impl(const std::vector<Path>& files) override;
+ Status exists_impl(const Path& path, bool* res) const override;
+ Status file_size_impl(const Path& file, size_t* file_size) const override;
+ Status list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) override;
+ Status rename_impl(const Path& orig_name, const Path& new_name) override;
+ Status rename_dir_impl(const Path& orig_name, const Path& new_name) override;
Status link_file_impl(const Path& src, const Path& dest);
- Status exists(const Path& path, bool* res) const override;
-
- Status exists_impl(const Path& path, bool* res) const;
-
- Status file_size(const Path& path, size_t* file_size) const override;
-
- Status file_size_impl(const Path& path, size_t* file_size) const;
-
- Status list(const Path& path, std::vector<Path>* files) override;
-
- Status list_impl(const Path& path, std::vector<Path>* files);
-
private:
LocalFileSystem(Path&& root_path, std::string&& id = "");
-
- Path absolute_path(const Path& path) const;
};
-const FileSystemSPtr& global_local_filesystem();
+const std::shared_ptr<LocalFileSystem>& global_local_filesystem();
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/local_file_writer.cpp b/be/src/io/fs/local_file_writer.cpp
index 85a577b958..c141471d98 100644
--- a/be/src/io/fs/local_file_writer.cpp
+++ b/be/src/io/fs/local_file_writer.cpp
@@ -56,21 +56,21 @@ Status sync_dir(const io::Path& dirname) {
namespace io {
-LocalFileWriter::LocalFileWriter(Path path, int fd, std::shared_ptr<LocalFileSystem> fs)
- : FileWriter(std::move(path)), _fd(fd), _fs(std::move(fs)) {
+LocalFileWriter::LocalFileWriter(Path path, int fd, FileSystemSPtr fs)
+ : FileWriter(std::move(path), fs), _fd(fd) {
+ _opened = true;
DorisMetrics::instance()->local_file_open_writing->increment(1);
DorisMetrics::instance()->local_file_writer_total->increment(1);
}
-LocalFileWriter::LocalFileWriter(Path path, int fd) : FileWriter(std::move(path)), _fd(fd) {
- DorisMetrics::instance()->local_file_open_writing->increment(1);
- DorisMetrics::instance()->local_file_writer_total->increment(1);
-}
+LocalFileWriter::LocalFileWriter(Path path, int fd)
+ : LocalFileWriter(path, fd, global_local_filesystem()) {}
LocalFileWriter::~LocalFileWriter() {
- if (!_closed) {
- WARN_IF_ERROR(abort(), fmt::format("Cannot abort {}", _path.native()));
+ if (_opened) {
+ close();
}
+ CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed;
}
Status LocalFileWriter::close() {
@@ -82,14 +82,6 @@ Status LocalFileWriter::abort() {
return io::global_local_filesystem()->delete_file(_path);
}
-Status LocalFileWriter::append(const Slice& data) {
- Status st = appendv(&data, 1);
- if (st.ok()) {
- DorisMetrics::instance()->local_bytes_written_total->increment(data.size);
- }
- return st;
-}
-
Status LocalFileWriter::appendv(const Slice* data, size_t data_cnt) {
DCHECK(!_closed);
_dirty = true;
@@ -142,6 +134,26 @@ Status LocalFileWriter::appendv(const Slice* data, size_t data_cnt) {
return Status::OK();
}
+Status LocalFileWriter::write_at(size_t offset, const Slice& data) {
+ DCHECK(!_closed);
+ _dirty = true;
+
+ size_t bytes_req = data.size;
+ char* from = data.data;
+
+ while (bytes_req != 0) {
+ auto res = ::pwrite(_fd, from, bytes_req, offset);
+ if (-1 == res && errno != EINTR) {
+ return Status::IOError("cannot write to {}: {}", _path.native(), std::strerror(errno));
+ }
+ if (res > 0) {
+ from += res;
+ bytes_req -= res;
+ }
+ }
+ return Status::OK();
+}
+
Status LocalFileWriter::finalize() {
DCHECK(!_closed);
if (_dirty) {
@@ -159,6 +171,7 @@ Status LocalFileWriter::_close(bool sync) {
if (_closed) {
return Status::OK();
}
+ _closed = true;
if (sync && _dirty) {
#ifdef __APPLE__
if (fcntl(_fd, F_FULLFSYNC) < 0) {
@@ -172,7 +185,6 @@ Status LocalFileWriter::_close(bool sync) {
RETURN_IF_ERROR(detail::sync_dir(_path.parent_path()));
_dirty = false;
}
- _closed = true;
DorisMetrics::instance()->local_file_open_writing->increment(-1);
DorisMetrics::instance()->file_created_total->increment(1);
@@ -184,25 +196,5 @@ Status LocalFileWriter::_close(bool sync) {
return Status::OK();
}
-Status LocalFileWriter::write_at(size_t offset, const Slice& data) {
- DCHECK(!_closed);
- _dirty = true;
-
- size_t bytes_req = data.size;
- char* from = data.data;
-
- while (bytes_req != 0) {
- auto res = ::pwrite(_fd, from, bytes_req, offset);
- if (-1 == res && errno != EINTR) {
- return Status::IOError("cannot write to {}: {}", _path.native(), std::strerror(errno));
- }
- if (res > 0) {
- from += res;
- bytes_req -= res;
- }
- }
- return Status::OK();
-}
-
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/local_file_writer.h b/be/src/io/fs/local_file_writer.h
index 4cce72b130..e4745893dd 100644
--- a/be/src/io/fs/local_file_writer.h
+++ b/be/src/io/fs/local_file_writer.h
@@ -28,37 +28,22 @@ namespace io {
class LocalFileWriter final : public FileWriter {
public:
- LocalFileWriter(Path path, int fd, std::shared_ptr<LocalFileSystem> fs);
-
+ LocalFileWriter(Path path, int fd, FileSystemSPtr fs);
LocalFileWriter(Path path, int fd);
-
~LocalFileWriter() override;
Status close() override;
-
Status abort() override;
-
- Status append(const Slice& data) override;
-
Status appendv(const Slice* data, size_t data_cnt) override;
-
Status write_at(size_t offset, const Slice& data) override;
-
Status finalize() override;
- size_t bytes_appended() const override { return _bytes_appended; }
-
- FileSystemSPtr fs() const override { return _fs; }
-
private:
Status _close(bool sync);
private:
int _fd; // owned
- std::shared_ptr<LocalFileSystem> _fs;
- size_t _bytes_appended = 0;
bool _dirty = false;
- bool _closed = false;
};
} // namespace io
diff --git a/be/src/io/fs/remote_file_system.cpp b/be/src/io/fs/remote_file_system.cpp
index 0222152e65..755de9a263 100644
--- a/be/src/io/fs/remote_file_system.cpp
+++ b/be/src/io/fs/remote_file_system.cpp
@@ -26,21 +26,91 @@
namespace doris {
namespace io {
-Status RemoteFileSystem::open_file(const Path& path, const FileReaderOptions& reader_options,
- FileReaderSPtr* reader, IOContext* io_ctx) {
+Status RemoteFileSystem::upload(const Path& local_file, const Path& dest_file) {
+ auto dest_path = absolute_path(dest_file);
if (bthread_self() == 0) {
- return open_file_impl(path, reader_options, reader, io_ctx);
+ return upload_impl(local_file, dest_path);
}
Status s;
- auto task = [&] { s = open_file_impl(path, reader_options, reader, io_ctx); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
+ auto task = [&] { s = upload_impl(local_file, dest_path); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status RemoteFileSystem::batch_upload(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) {
+ std::vector<Path> remote_paths;
+ for (auto& path : remote_files) {
+ remote_paths.push_back(absolute_path(path));
+ }
+ if (bthread_self() == 0) {
+ return batch_upload_impl(local_files, remote_paths);
+ }
+ Status s;
+ auto task = [&] { s = batch_upload_impl(local_files, remote_paths); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status RemoteFileSystem::direct_upload(const Path& remote_file, const std::string& content) {
+ auto remote_path = absolute_path(remote_file);
+ if (bthread_self() == 0) {
+ return direct_upload_impl(remote_path, content);
+ }
+ Status s;
+ auto task = [&] { s = direct_upload_impl(remote_path, content); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status RemoteFileSystem::upload_with_checksum(const Path& local_file, const Path& remote,
+ const std::string& checksum) {
+ auto remote_path = absolute_path(remote);
+ if (bthread_self() == 0) {
+ return upload_with_checksum_impl(local_file, remote_path, checksum);
+ }
+ Status s;
+ auto task = [&] { s = upload_with_checksum_impl(local_file, remote_path, checksum); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status RemoteFileSystem::download(const Path& remote_file, const Path& local) {
+ auto remote_path = absolute_path(remote_file);
+ if (bthread_self() == 0) {
+ return download_impl(remote_path, local);
+ }
+ Status s;
+ auto task = [&] { s = download_impl(remote_path, local); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status RemoteFileSystem::direct_download(const Path& remote_file, std::string* content) {
+ auto remote_path = absolute_path(remote_file);
+ if (bthread_self() == 0) {
+ return direct_download_impl(remote_path, content);
+ }
+ Status s;
+ auto task = [&] { s = direct_download_impl(remote_path, content); };
+ AsyncIO::run_task(task, _type);
+ return s;
+}
+
+Status RemoteFileSystem::connect() {
+ if (bthread_self() == 0) {
+ return connect_impl();
+ }
+ Status s;
+ auto task = [&] { s = connect_impl(); };
+ AsyncIO::run_task(task, _type);
return s;
}
Status RemoteFileSystem::open_file_impl(const Path& path, const FileReaderOptions& reader_options,
- FileReaderSPtr* reader, IOContext* io_ctx) {
+ FileReaderSPtr* reader) {
FileReaderSPtr raw_reader;
- RETURN_IF_ERROR(open_file(path, &raw_reader, io_ctx));
+ RETURN_IF_ERROR(open_file_internal(path, &raw_reader));
switch (reader_options.cache_type) {
case io::FileCachePolicy::NO_CACHE: {
*reader = raw_reader;
@@ -57,11 +127,9 @@ Status RemoteFileSystem::open_file_impl(const Path& path, const FileReaderOption
break;
}
case io::FileCachePolicy::FILE_BLOCK_CACHE: {
- DCHECK(io_ctx);
StringPiece str(raw_reader->path().native());
std::string cache_path = reader_options.path_policy.get_cache_path(path.native());
- *reader =
- std::make_shared<CachedRemoteFileReader>(std::move(raw_reader), cache_path, io_ctx);
+ *reader = std::make_shared<CachedRemoteFileReader>(std::move(raw_reader), cache_path);
break;
}
default: {
diff --git a/be/src/io/fs/remote_file_system.h b/be/src/io/fs/remote_file_system.h
index ff1d2f6961..a62746afd5 100644
--- a/be/src/io/fs/remote_file_system.h
+++ b/be/src/io/fs/remote_file_system.h
@@ -28,28 +28,53 @@ public:
: FileSystem(std::move(root_path), std::move(id), type) {}
~RemoteFileSystem() override = default;
- // `local_path` should be an absolute path on local filesystem.
- virtual Status upload(const Path& local_path, const Path& dest_path) = 0;
+ Status upload(const Path& local_file, const Path& dest_file);
+ Status batch_upload(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files);
+ Status direct_upload(const Path& remote_file, const std::string& content);
+ Status upload_with_checksum(const Path& local_file, const Path& remote,
+ const std::string& checksum);
+ Status download(const Path& remote_file, const Path& local);
+ Status direct_download(const Path& remote_file, std::string* content);
- virtual Status batch_upload(const std::vector<Path>& local_paths,
- const std::vector<Path>& dest_paths) = 0;
+ Status connect();
- virtual Status batch_delete(const std::vector<Path>& paths) {
- return Status::NotSupported("batch_delete");
- }
+protected:
+ /// connect to remote file system
+ virtual Status connect_impl() = 0;
- virtual Status connect() = 0;
+ virtual Status open_file_impl(const Path& file, const FileReaderOptions& reader_options,
+ FileReaderSPtr* reader) override;
+ /// upload load_file to remote remote_file
+ /// local_file should be an absolute path on local filesystem.
+ virtual Status upload_impl(const Path& local_file, const Path& remote_file) = 0;
- Status open_file(const Path& path, const FileReaderOptions& reader_options,
- FileReaderSPtr* reader, IOContext* io_ctx) override;
+ /// upload all files in load_files to remote_files
+ /// path in local_files should be an absolute path on local filesystem.
+ /// the size of local_files and remote_files must be equal.
+ virtual Status batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) = 0;
- Status open_file_impl(const Path& path, const FileReaderOptions& reader_options,
- FileReaderSPtr* reader, IOContext* io_ctx);
+ /// save the content in "content" directly to remote file
+ virtual Status direct_upload_impl(const Path& remote_file, const std::string& content) = 0;
- Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override {
- return Status::NotSupported("implemented in derived classes");
- }
+ /// upload local_file to remote_file,
+ /// and the final remote file name is "remote_file.checksum"
+ virtual Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file,
+ const std::string& checksum) = 0;
+
+ /// download remote_file to local_file
+ /// local_file should be an absolute path on local filesystem.
+ virtual Status download_impl(const Path& remote_file, const Path& local_file) = 0;
+
+ /// save of content of remote_file directly into "content"
+ virtual Status direct_download_impl(const Path& remote_file, std::string* content) = 0;
+
+ // The derived class should implement this method.
+ virtual Status open_file_internal(const Path& file, FileReaderSPtr* reader) = 0;
};
+using RemoteFileSystemSPtr = std::shared_ptr<RemoteFileSystem>;
+
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp
index 3b6ce94034..b60e038d39 100644
--- a/be/src/io/fs/s3_file_reader.cpp
+++ b/be/src/io/fs/s3_file_reader.cpp
@@ -50,19 +50,8 @@ Status S3FileReader::close() {
return Status::OK();
}
-Status S3FileReader::read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) {
- if (bthread_self() == 0) {
- return read_at_impl(offset, result, io_ctx, bytes_read);
- }
- Status s;
- auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileReader::read_at_impl(size_t offset, Slice result, const IOContext& /*io_ctx*/,
- size_t* bytes_read) {
+Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) {
DCHECK(!closed());
if (offset > _file_size) {
return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})",
diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h
index 9ab21c9f48..c21084ae8d 100644
--- a/be/src/io/fs/s3_file_reader.h
+++ b/be/src/io/fs/s3_file_reader.h
@@ -34,11 +34,6 @@ public:
Status close() override;
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
- Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read);
-
const Path& path() const override { return _path; }
size_t size() const override { return _file_size; }
@@ -47,6 +42,10 @@ public:
FileSystemSPtr fs() const override { return _fs; }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
Path _path;
size_t _file_size;
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index ee2161975f..d970c9dc4e 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -39,20 +39,34 @@
#include "io/fs/remote_file_system.h"
#include "io/fs/s3_file_reader.h"
#include "io/fs/s3_file_writer.h"
-#include "util/async_io.h"
+#include "util/s3_uri.h"
+#include "util/s3_util.h"
namespace doris {
namespace io {
#ifndef CHECK_S3_CLIENT
-#define CHECK_S3_CLIENT(client) \
- if (!client) { \
- return Status::InternalError("init s3 client error"); \
+#define CHECK_S3_CLIENT(client) \
+ if (!client) { \
+ return Status::IOError("init s3 client error"); \
}
#endif
-std::shared_ptr<S3FileSystem> S3FileSystem::create(S3Conf s3_conf, std::string id) {
- return std::shared_ptr<S3FileSystem>(new S3FileSystem(std::move(s3_conf), std::move(id)));
+#ifndef CHECK_S3_PATH
+#define CHECK_S3_PATH(uri, path) \
+ S3URI uri(path.string()); \
+ RETURN_IF_ERROR(uri.parse());
+#endif
+
+#ifndef GET_KEY
+#define GET_KEY(key, path) \
+ std::string key; \
+ RETURN_IF_ERROR(get_key(path, &key));
+#endif
+
+Status S3FileSystem::create(S3Conf s3_conf, std::string id, std::shared_ptr<S3FileSystem>* fs) {
+ (*fs).reset(new S3FileSystem(std::move(s3_conf), std::move(id)));
+ return (*fs)->connect();
}
S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id)
@@ -60,11 +74,14 @@ S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id)
fmt::format("{}/{}/{}", s3_conf.endpoint, s3_conf.bucket, s3_conf.prefix),
std::move(id), FileSystemType::S3),
_s3_conf(std::move(s3_conf)) {
- if (_s3_conf.prefix.size() > 0 && _s3_conf.prefix[0] == '/') {
- _s3_conf.prefix = _s3_conf.prefix.substr(1);
- }
- if (!_s3_conf.prefix.empty() && _s3_conf.prefix.back() == '/') {
- _s3_conf.prefix.pop_back();
+ // remove the first and last '/'
+ if (!_s3_conf.prefix.empty()) {
+ if (_s3_conf.prefix[0] == '/') {
+ _s3_conf.prefix = _s3_conf.prefix.substr(1);
+ }
+ if (_s3_conf.prefix.back() == '/') {
+ _s3_conf.prefix.pop_back();
+ }
}
_executor = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(
id.c_str(), config::s3_transfer_executor_pool_size);
@@ -72,142 +89,25 @@ S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id)
S3FileSystem::~S3FileSystem() = default;
-Status S3FileSystem::connect() {
- if (bthread_self() == 0) {
- return connect_impl();
- }
- Status s;
- auto task = [&] { s = connect_impl(); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
Status S3FileSystem::connect_impl() {
std::lock_guard lock(_client_mu);
- _client = ClientFactory::instance().create(_s3_conf);
+ _client = S3ClientFactory::instance().create(_s3_conf);
if (!_client) {
- return Status::InternalError("failed to init s3 client with {}", _s3_conf.to_string());
+ return Status::IOError("failed to init s3 client with {}", _s3_conf.to_string());
}
return Status::OK();
}
-Status S3FileSystem::upload(const Path& local_path, const Path& dest_path) {
- if (bthread_self() == 0) {
- return upload_impl(local_path, dest_path);
- }
- Status s;
- auto task = [&] { s = upload_impl(local_path, dest_path); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileSystem::upload_impl(const Path& local_path, const Path& dest_path) {
- auto client = get_client();
- CHECK_S3_CLIENT(client);
-
- Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get());
- transfer_config.s3Client = client;
- auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config);
-
- auto start = std::chrono::steady_clock::now();
-
- auto key = get_key(dest_path);
- auto handle = transfer_manager->UploadFile(local_path.native(), _s3_conf.bucket, key,
- "text/plain", Aws::Map<Aws::String, Aws::String>());
- handle->WaitUntilFinished();
-
- auto duration = std::chrono::duration<float>(std::chrono::steady_clock::now() - start);
-
- if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) {
- return Status::IOError("failed to upload(endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, key,
- handle->GetLastError().GetMessage());
- }
-
- auto file_size = std::filesystem::file_size(local_path);
- LOG(INFO) << "Upload " << local_path.native() << " to s3, endpoint=" << _s3_conf.endpoint
- << ", bucket=" << _s3_conf.bucket << ", key=" << key
- << ", duration=" << duration.count() << ", capacity=" << file_size
- << ", tp=" << (file_size) / duration.count();
-
+Status S3FileSystem::create_file_impl(const Path& file, FileWriterPtr* writer) {
+ GET_KEY(key, file);
+ *writer = std::make_unique<S3FileWriter>(key, get_client(), _s3_conf, getSPtr());
return Status::OK();
}
-Status S3FileSystem::batch_upload(const std::vector<Path>& local_paths,
- const std::vector<Path>& dest_paths) {
- if (bthread_self() == 0) {
- return batch_upload_impl(local_paths, dest_paths);
- }
- Status s;
- auto task = [&] { s = batch_upload_impl(local_paths, dest_paths); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileSystem::batch_upload_impl(const std::vector<Path>& local_paths,
- const std::vector<Path>& dest_paths) {
- auto client = get_client();
- CHECK_S3_CLIENT(client);
-
- if (local_paths.size() != dest_paths.size()) {
- return Status::InvalidArgument("local_paths.size() != dest_paths.size()");
- }
-
- Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get());
- transfer_config.s3Client = client;
- auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config);
-
- std::vector<std::shared_ptr<Aws::Transfer::TransferHandle>> handles;
- for (int i = 0; i < local_paths.size(); ++i) {
- auto key = get_key(dest_paths[i]);
- LOG(INFO) << "Start to upload " << local_paths[i].native()
- << " to s3, endpoint=" << _s3_conf.endpoint << ", bucket=" << _s3_conf.bucket
- << ", key=" << key;
- auto handle =
- transfer_manager->UploadFile(local_paths[i].native(), _s3_conf.bucket, key,
- "text/plain", Aws::Map<Aws::String, Aws::String>());
- handles.push_back(std::move(handle));
- }
- for (auto& handle : handles) {
- handle->WaitUntilFinished();
- if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) {
- // TODO(cyx): Maybe we can cancel remaining handles.
- return Status::IOError(handle->GetLastError().GetMessage());
- }
- }
- return Status::OK();
-}
-
-Status S3FileSystem::create_file(const Path& path, FileWriterPtr* writer) {
- if (bthread_self() == 0) {
- return create_file_impl(path, writer);
- }
- Status s;
- auto task = [&] { s = create_file_impl(path, writer); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileSystem::create_file_impl(const Path& path, FileWriterPtr* writer) {
- *writer = std::make_unique<S3FileWriter>(Path(get_key(path)), get_client(), _s3_conf);
- return Status::OK();
-}
-
-Status S3FileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) {
- if (bthread_self() == 0) {
- return open_file_impl(path, reader, io_ctx);
- }
- Status s;
- auto task = [&] { s = open_file_impl(path, reader, io_ctx); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileSystem::open_file_impl(const Path& path, FileReaderSPtr* reader,
- IOContext* /*io_ctx*/) {
+Status S3FileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader) {
size_t fsize = 0;
- RETURN_IF_ERROR(file_size(path, &fsize));
- auto key = get_key(path);
+ RETURN_IF_ERROR(file_size_impl(file, &fsize));
+ GET_KEY(key, file);
auto fs_path = Path(_s3_conf.endpoint) / _s3_conf.bucket / key;
*reader = std::make_shared<S3FileReader>(
std::move(fs_path), fsize, std::move(key), _s3_conf.bucket,
@@ -215,22 +115,16 @@ Status S3FileSystem::open_file_impl(const Path& path, FileReaderSPtr* reader,
return Status::OK();
}
-Status S3FileSystem::delete_file(const Path& path) {
- if (bthread_self() == 0) {
- return delete_file_impl(path);
- }
- Status s;
- auto task = [&] { s = delete_file_impl(path); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
+Status S3FileSystem::create_directory_impl(const Path& dir) {
+ return Status::OK();
}
-Status S3FileSystem::delete_file_impl(const Path& path) {
+Status S3FileSystem::delete_file_impl(const Path& file) {
auto client = get_client();
CHECK_S3_CLIENT(client);
Aws::S3::Model::DeleteObjectRequest request;
- auto key = get_key(path);
+ GET_KEY(key, file);
request.WithBucket(_s3_conf.bucket).WithKey(key);
auto outcome = client->DeleteObject(request);
@@ -238,31 +132,15 @@ Status S3FileSystem::delete_file_impl(const Path& path) {
outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) {
return Status::OK();
}
- return Status::IOError("failed to delete object(endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, key,
- outcome.GetError().GetMessage());
-}
-
-Status S3FileSystem::create_directory(const Path& path) {
- return Status::OK();
+ return Status::IOError("failed to delete file {}: {}", file.native(), error_msg(key, outcome));
}
-Status S3FileSystem::delete_directory(const Path& path) {
- if (bthread_self() == 0) {
- return delete_directory_impl(path);
- }
- Status s;
- auto task = [&] { s = delete_directory_impl(path); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileSystem::delete_directory_impl(const Path& path) {
+Status S3FileSystem::delete_directory_impl(const Path& dir) {
auto client = get_client();
CHECK_S3_CLIENT(client);
Aws::S3::Model::ListObjectsV2Request request;
- auto prefix = get_key(path);
+ GET_KEY(prefix, dir);
if (!prefix.empty() && prefix.back() != '/') {
prefix.push_back('/');
}
@@ -274,9 +152,8 @@ Status S3FileSystem::delete_directory_impl(const Path& path) {
do {
auto outcome = client->ListObjectsV2(request);
if (!outcome.IsSuccess()) {
- return Status::IOError("failed to list objects(endpoint={}, bucket={}, prefix={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, prefix,
- outcome.GetError().GetMessage());
+ return Status::IOError("failed to list objects when delete dir {}: {}", dir.native(),
+ error_msg(prefix, outcome));
}
const auto& result = outcome.GetResult();
Aws::Vector<Aws::S3::Model::ObjectIdentifier> objects;
@@ -290,16 +167,13 @@ Status S3FileSystem::delete_directory_impl(const Path& path) {
delete_request.SetDelete(std::move(del));
auto delete_outcome = client->DeleteObjects(delete_request);
if (!delete_outcome.IsSuccess()) {
- return Status::IOError(
- "failed to delete objects(endpoint={}, bucket={}, prefix={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, prefix,
- delete_outcome.GetError().GetMessage());
+ return Status::IOError("failed to delete dir {}: {}", dir.native(),
+ error_msg(prefix, delete_outcome));
}
if (!delete_outcome.GetResult().GetErrors().empty()) {
const auto& e = delete_outcome.GetResult().GetErrors().front();
- return Status::IOError("fail to delete object(endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, e.GetKey(),
- e.GetMessage());
+ return Status::IOError("fail to delete object: {}",
+ error_msg(e.GetKey(), e.GetMessage()));
}
VLOG_TRACE << "delete " << objects.size()
<< " s3 objects, endpoint: " << _s3_conf.endpoint
@@ -311,26 +185,51 @@ Status S3FileSystem::delete_directory_impl(const Path& path) {
return Status::OK();
}
-Status S3FileSystem::link_file(const Path& src, const Path& dest) {
- return Status::NotSupported("not support");
-}
+Status S3FileSystem::batch_delete_impl(const std::vector<Path>& remote_files) {
+ auto client = get_client();
+ CHECK_S3_CLIENT(client);
-Status S3FileSystem::exists(const Path& path, bool* res) const {
- if (bthread_self() == 0) {
- return exists_impl(path, res);
- }
- Status s;
- auto task = [&] { s = exists_impl(path, res); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
+ // `DeleteObjectsRequest` can only contain 1000 keys at most.
+ constexpr size_t max_delete_batch = 1000;
+ auto path_iter = remote_files.begin();
+
+ Aws::S3::Model::DeleteObjectsRequest delete_request;
+ delete_request.SetBucket(_s3_conf.bucket);
+ do {
+ Aws::S3::Model::Delete del;
+ Aws::Vector<Aws::S3::Model::ObjectIdentifier> objects;
+ auto path_begin = path_iter;
+ for (; path_iter != remote_files.end() && (path_iter - path_begin < max_delete_batch);
+ ++path_iter) {
+ GET_KEY(key, *path_iter);
+ objects.emplace_back().SetKey(key);
+ }
+ if (objects.empty()) {
+ return Status::OK();
+ }
+ del.WithObjects(std::move(objects)).SetQuiet(true);
+ delete_request.SetDelete(std::move(del));
+ auto delete_outcome = client->DeleteObjects(delete_request);
+ if (UNLIKELY(!delete_outcome.IsSuccess())) {
+ return Status::IOError("failed to delete objects: {}",
+ error_msg(objects.front().GetKey(), delete_outcome));
+ }
+ if (UNLIKELY(!delete_outcome.GetResult().GetErrors().empty())) {
+ const auto& e = delete_outcome.GetResult().GetErrors().front();
+ return Status::IOError("failed to delete objects: {}",
+ error_msg(e.GetKey(), delete_outcome));
+ }
+ } while (path_iter != remote_files.end());
+
+ return Status::OK();
}
Status S3FileSystem::exists_impl(const Path& path, bool* res) const {
auto client = get_client();
CHECK_S3_CLIENT(client);
+ GET_KEY(key, path);
Aws::S3::Model::HeadObjectRequest request;
- auto key = get_key(path);
request.WithBucket(_s3_conf.bucket).WithKey(key);
auto outcome = client->HeadObject(request);
@@ -339,62 +238,64 @@ Status S3FileSystem::exists_impl(const Path& path, bool* res) const {
} else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) {
*res = false;
} else {
- return Status::IOError("failed to get object head(endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, key,
- outcome.GetError().GetMessage());
+ return Status::IOError("failed to check exists {}: {}", path.native(),
+ error_msg(key, outcome));
}
return Status::OK();
}
-Status S3FileSystem::file_size(const Path& path, size_t* file_size) const {
- if (bthread_self() == 0) {
- return file_size_impl(path, file_size);
- }
- Status s;
- auto task = [&] { s = file_size_impl(path, file_size); };
- AsyncIO::run_task(task, io::FileSystemType::S3);
- return s;
-}
-
-Status S3FileSystem::file_size_impl(const Path& path, size_t* file_size) const {
+Status S3FileSystem::file_size_impl(const Path& file, size_t* file_size) const {
auto client = get_client();
CHECK_S3_CLIENT(client);
Aws::S3::Model::HeadObjectRequest request;
- auto key = get_key(path);
+ GET_KEY(key, file);
request.WithBucket(_s3_conf.bucket).WithKey(key);
auto outcome = client->HeadObject(request);
if (outcome.IsSuccess()) {
*file_size = outcome.GetResult().GetContentLength();
} else {
- return Status::IOError("failed to get object size(endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, key,
- outcome.GetError().GetMessage());
+ return Status::IOError("failed to get file size {}, {}", file.native(),
+ error_msg(key, outcome));
}
return Status::OK();
}
-Status S3FileSystem::list(const Path& path, std::vector<Path>* files) {
+Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) {
+ // For object storage, this path is always not exist.
+ // So we ignore this property and set exists to true.
+ *exists = true;
auto client = get_client();
CHECK_S3_CLIENT(client);
-
- Aws::S3::Model::ListObjectsV2Request request;
- auto prefix = get_key(path);
+ GET_KEY(prefix, dir);
if (!prefix.empty() && prefix.back() != '/') {
prefix.push_back('/');
}
+
+ Aws::S3::Model::ListObjectsV2Request request;
request.WithBucket(_s3_conf.bucket).WithPrefix(prefix);
bool is_trucated = false;
do {
auto outcome = client->ListObjectsV2(request);
if (!outcome.IsSuccess()) {
- return Status::IOError("failed to list objects(endpoint={}, bucket={}, prefix={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, prefix,
- outcome.GetError().GetMessage());
+ return Status::IOError("failed to list {}: {}", dir.native(),
+ error_msg(prefix, outcome));
}
for (const auto& obj : outcome.GetResult().GetContents()) {
- files->push_back(obj.GetKey().substr(prefix.size()));
+ std::string key = obj.GetKey();
+ bool is_dir = (key.at(key.size() - 1) == '/');
+ if (only_file && is_dir) {
+ continue;
+ }
+ FileInfo file_info;
+ // note: if full path is s3://bucket/path/to/file.txt
+ // obj.GetKey() will be /path/to/file.txt
+ file_info.file_name = obj.GetKey().substr(prefix.size());
+ file_info.file_size = obj.GetSize();
+ file_info.is_file = !is_dir;
+ files->push_back(std::move(file_info));
}
is_trucated = outcome.GetResult().GetIsTruncated();
request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken());
@@ -402,54 +303,203 @@ Status S3FileSystem::list(const Path& path, std::vector<Path>* files) {
return Status::OK();
}
-Status S3FileSystem::batch_delete(const std::vector<Path>& paths) {
+Status S3FileSystem::rename_impl(const Path& orig_name, const Path& new_name) {
+ RETURN_IF_ERROR(copy(orig_name, new_name));
+ return delete_file_impl(orig_name);
+}
+
+Status S3FileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) {
+ RETURN_IF_ERROR(copy_dir(orig_name, new_name));
+ return delete_directory_impl(orig_name);
+}
+
+Status S3FileSystem::upload_impl(const Path& local_file, const Path& remote_file) {
auto client = get_client();
CHECK_S3_CLIENT(client);
- // `DeleteObjectsRequest` can only contain 1000 keys at most.
- constexpr size_t max_delete_batch = 1000;
- auto path_iter = paths.begin();
+ Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get());
+ transfer_config.s3Client = client;
+ auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config);
- Aws::S3::Model::DeleteObjectsRequest delete_request;
- delete_request.SetBucket(_s3_conf.bucket);
- do {
- Aws::S3::Model::Delete del;
- Aws::Vector<Aws::S3::Model::ObjectIdentifier> objects;
- auto path_begin = path_iter;
- for (; path_iter != paths.end() && (path_iter - path_begin < max_delete_batch);
- ++path_iter) {
- objects.emplace_back().SetKey(get_key(*path_iter));
- }
- if (objects.empty()) {
- return Status::OK();
+ auto start = std::chrono::steady_clock::now();
+
+ GET_KEY(key, remote_file);
+ auto handle = transfer_manager->UploadFile(local_file.native(), _s3_conf.bucket, key,
+ "text/plain", Aws::Map<Aws::String, Aws::String>());
+ handle->WaitUntilFinished();
+
+ auto duration = std::chrono::duration<float>(std::chrono::steady_clock::now() - start);
+
+ if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) {
+ return Status::IOError("failed to upload {}: {}", remote_file.native(),
+ error_msg(key, handle->GetLastError().GetMessage()));
+ }
+
+ auto file_size = std::filesystem::file_size(local_file);
+ LOG(INFO) << "Upload " << local_file.native() << " to s3, endpoint=" << _s3_conf.endpoint
+ << ", bucket=" << _s3_conf.bucket << ", key=" << key
+ << ", duration=" << duration.count() << ", capacity=" << file_size
+ << ", tp=" << (file_size) / duration.count();
+
+ return Status::OK();
+}
+
+Status S3FileSystem::batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) {
+ auto client = get_client();
+ CHECK_S3_CLIENT(client);
+
+ if (local_files.size() != remote_files.size()) {
+ return Status::InvalidArgument("local_files.size({}) != remote_files.size({})",
+ local_files.size(), remote_files.size());
+ }
+
+ Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get());
+ transfer_config.s3Client = client;
+ auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config);
+
+ std::vector<std::shared_ptr<Aws::Transfer::TransferHandle>> handles;
+ for (int i = 0; i < local_files.size(); ++i) {
+ GET_KEY(key, remote_files[i]);
+ LOG(INFO) << "Start to upload " << local_files[i].native()
+ << " to s3, endpoint=" << _s3_conf.endpoint << ", bucket=" << _s3_conf.bucket
+ << ", key=" << key;
+ auto handle =
+ transfer_manager->UploadFile(local_files[i].native(), _s3_conf.bucket, key,
+ "text/plain", Aws::Map<Aws::String, Aws::String>());
+ handles.push_back(std::move(handle));
+ }
+ for (auto& handle : handles) {
+ handle->WaitUntilFinished();
+ if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) {
+ // TODO(cyx): Maybe we can cancel remaining handles.
+ return Status::IOError("failed to upload: {}",
+ error_msg("", handle->GetLastError().GetMessage()));
}
- del.WithObjects(std::move(objects)).SetQuiet(true);
- delete_request.SetDelete(std::move(del));
- auto delete_outcome = client->DeleteObjects(delete_request);
- if (UNLIKELY(!delete_outcome.IsSuccess())) {
- return Status::IOError(
- "failed to delete objects(endpoint={}, bucket={}, key[0]={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, objects.front().GetKey(),
- delete_outcome.GetError().GetMessage());
+ }
+ return Status::OK();
+}
+
+Status S3FileSystem::direct_upload_impl(const Path& remote_file, const std::string& content) {
+ CHECK_S3_CLIENT(_client);
+ Aws::S3::Model::PutObjectRequest request;
+ GET_KEY(key, remote_file);
+ request.WithBucket(_s3_conf.bucket).WithKey(key);
+ const std::shared_ptr<Aws::IOStream> input_data =
+ Aws::MakeShared<Aws::StringStream>("upload_directly");
+ *input_data << content.c_str();
+ if (input_data->good()) {
+ request.SetBody(input_data);
+ }
+ if (!input_data->good()) {
+ return Status::IOError("failed to direct upload {}: failed to read from string",
+ remote_file.native());
+ }
+ Aws::S3::Model::PutObjectOutcome response = _client->PutObject(request);
+ if (response.IsSuccess()) {
+ return Status::OK();
+ } else {
+ return Status::IOError("failed to direct upload {}: {}", remote_file.native(),
+ error_msg(key, response));
+ }
+}
+
+Status S3FileSystem::upload_with_checksum_impl(const Path& local_file, const Path& remote_file,
+ const std::string& checksum) {
+ return upload_impl(local_file, remote_file.string() + "." + checksum);
+}
+
+Status S3FileSystem::download_impl(const Path& remote_file, const Path& local_file) {
+ auto client = get_client();
+ CHECK_S3_CLIENT(client);
+ GET_KEY(key, remote_file);
+ Aws::S3::Model::GetObjectRequest request;
+ request.WithBucket(_s3_conf.bucket).WithKey(key);
+ Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request);
+ if (response.IsSuccess()) {
+ Aws::OFStream local_file_s;
+ local_file_s.open(local_file, std::ios::out | std::ios::binary);
+ if (local_file_s.good()) {
+ local_file_s << response.GetResult().GetBody().rdbuf();
}
- if (UNLIKELY(!delete_outcome.GetResult().GetErrors().empty())) {
- const auto& e = delete_outcome.GetResult().GetErrors().front();
- return Status::IOError("failed to delete objects(endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, e.GetKey(),
- delete_outcome.GetError().GetMessage());
+ if (!local_file_s.good()) {
+ return Status::IOError("failed to download {}: failed to write file: {}",
+ remote_file.native(), local_file.native());
}
- } while (path_iter != paths.end());
+ } else {
+ return Status::IOError("failed to download {}: {}", remote_file.native(),
+ error_msg(key, response));
+ }
+ return Status::OK();
+}
+Status S3FileSystem::direct_download_impl(const Path& remote, std::string* content) {
+ CHECK_S3_CLIENT(_client);
+ Aws::S3::Model::GetObjectRequest request;
+ GET_KEY(key, remote);
+ request.WithBucket(_s3_conf.bucket).WithKey(key);
+ Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request);
+ if (response.IsSuccess()) {
+ std::stringstream ss;
+ ss << response.GetResult().GetBody().rdbuf();
+ *content = ss.str();
+ } else {
+ return Status::IOError("failed to direct download {}: {}", remote.native(),
+ error_msg(key, response));
+ }
return Status::OK();
}
-std::string S3FileSystem::get_key(const Path& path) const {
- StringPiece str(path.native());
- if (str.starts_with(_root_path.native())) {
- return fmt::format("{}/{}", _s3_conf.prefix, str.data() + _root_path.native().size());
+Status S3FileSystem::copy(const Path& src, const Path& dst) {
+ CHECK_S3_CLIENT(_client);
+ Aws::S3::Model::CopyObjectRequest request;
+ GET_KEY(src_key, src);
+ GET_KEY(dst_key, dst);
+ request.WithCopySource(_s3_conf.bucket + "/" + src_key)
+ .WithKey(dst_key)
+ .WithBucket(_s3_conf.bucket);
+ Aws::S3::Model::CopyObjectOutcome response = _client->CopyObject(request);
+ if (response.IsSuccess()) {
+ return Status::OK();
+ } else {
+ return Status::IOError("failed to copy from {} to {}: {}", src.native(), dst.native(),
+ error_msg(src_key, response));
}
- // We consider it as a relative path.
- return fmt::format("{}/{}", _s3_conf.prefix, path.native());
+}
+
+Status S3FileSystem::copy_dir(const Path& src, const Path& dst) {
+ std::vector<FileInfo> files;
+ bool exists = false;
+ RETURN_IF_ERROR(list_impl(src, true, &files, &exists));
+ if (!exists) {
+ return Status::IOError("path not found: {}", src.native());
+ }
+ if (files.empty()) {
+ LOG(WARNING) << "Nothing need to copy: " << src << " -> " << dst;
+ return Status::OK();
+ }
+ for (auto& file : files) {
+ RETURN_IF_ERROR(copy(src / file.file_name, dst / file.file_name));
+ }
+ return Status::OK();
+}
+
+Status S3FileSystem::get_key(const Path& path, std::string* key) const {
+ CHECK_S3_PATH(uri, path);
+ *key = uri.get_key();
+ return Status::OK();
+}
+
+template <typename AwsOutcome>
+std::string S3FileSystem::error_msg(const std::string& key, const AwsOutcome& outcome) const {
+ return fmt::format("(endpoint: {}, bucket: {}, key:{}, {}), {}", _s3_conf.endpoint,
+ _s3_conf.bucket, key, outcome.GetError().GetExceptionName(),
+ outcome.GetError().GetMessage());
+}
+
+std::string S3FileSystem::error_msg(const std::string& key, const std::string& err) const {
+ return fmt::format("(endpoint: {}, bucket: {}, key:{}), {}", _s3_conf.endpoint, _s3_conf.bucket,
+ key, err);
}
} // namespace io
diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h
index 0576421e53..f97e9137cc 100644
--- a/be/src/io/fs/s3_file_system.h
+++ b/be/src/io/fs/s3_file_system.h
@@ -35,77 +35,62 @@ namespace io {
// This class is thread-safe.(Except `set_xxx` method)
class S3FileSystem final : public RemoteFileSystem {
public:
- static std::shared_ptr<S3FileSystem> create(S3Conf s3_conf, std::string id);
-
+ static Status create(S3Conf s3_conf, std::string id, std::shared_ptr<S3FileSystem>* fs);
~S3FileSystem() override;
-
- Status create_file(const Path& path, FileWriterPtr* writer) override;
-
- Status create_file_impl(const Path& path, FileWriterPtr* writer);
-
- Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override;
-
- Status open_file_impl(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx);
-
- Status delete_file(const Path& path) override;
-
- Status delete_file_impl(const Path& path);
-
- Status create_directory(const Path& path) override;
-
- // Delete all objects start with path.
- Status delete_directory(const Path& path) override;
-
- Status delete_directory_impl(const Path& path);
-
- Status link_file(const Path& src, const Path& dest) override;
-
- Status exists(const Path& path, bool* res) const override;
-
- Status exists_impl(const Path& path, bool* res) const;
-
- Status file_size(const Path& path, size_t* file_size) const override;
-
- Status file_size_impl(const Path& path, size_t* file_size) const;
-
- Status list(const Path& path, std::vector<Path>* files) override;
-
- Status upload(const Path& local_path, const Path& dest_path) override;
-
- Status upload_impl(const Path& local_path, const Path& dest_path);
-
- Status batch_upload(const std::vector<Path>& local_paths,
- const std::vector<Path>& dest_paths) override;
-
- Status batch_upload_impl(const std::vector<Path>& local_paths,
- const std::vector<Path>& dest_paths);
-
- Status batch_delete(const std::vector<Path>& paths) override;
-
- Status connect() override;
-
- Status connect_impl();
+ // Guarded by external lock.
+ void set_conf(S3Conf s3_conf) { _s3_conf = std::move(s3_conf); }
std::shared_ptr<Aws::S3::S3Client> get_client() const {
std::lock_guard lock(_client_mu);
return _client;
}
- // Guarded by external lock.
- void set_conf(S3Conf s3_conf) { _s3_conf = std::move(s3_conf); }
-
- std::string get_key(const Path& path) const;
+protected:
+ Status connect_impl() override;
+ Status create_file_impl(const Path& file, FileWriterPtr* writer) override;
+ Status open_file_internal(const Path& file, FileReaderSPtr* reader) override;
+ Status create_directory_impl(const Path& dir) override;
+ Status delete_file_impl(const Path& file) override;
+ Status delete_directory_impl(const Path& dir) override;
+ Status batch_delete_impl(const std::vector<Path>& files) override;
+ Status exists_impl(const Path& path, bool* res) const override;
+ Status file_size_impl(const Path& file, size_t* file_size) const override;
+ Status list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) override;
+ Status rename_impl(const Path& orig_name, const Path& new_name) override;
+ Status rename_dir_impl(const Path& orig_name, const Path& new_name) override;
+
+ Status upload_impl(const Path& local_file, const Path& remote_file) override;
+ Status batch_upload_impl(const std::vector<Path>& local_files,
+ const std::vector<Path>& remote_files) override;
+ Status direct_upload_impl(const Path& remote_file, const std::string& content) override;
+ Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file,
+ const std::string& checksum) override;
+ Status download_impl(const Path& remote_file, const Path& local_file) override;
+ Status direct_download_impl(const Path& remote_file, std::string* content) override;
+
+ Path absolute_path(const Path& path) const override {
+ // do nothing
+ return path;
+ }
private:
S3FileSystem(S3Conf&& s3_conf, std::string&& id);
+ template <typename AwsOutcome>
+ std::string error_msg(const std::string& key, const AwsOutcome& outcome) const;
+ std::string error_msg(const std::string& key, const std::string& err) const;
+ /// copy file from src to dst
+ Status copy(const Path& src, const Path& dst);
+ /// copy dir from src to dst
+ Status copy_dir(const Path& src, const Path& dst);
+ Status get_key(const Path& path, std::string* key) const;
+
private:
S3Conf _s3_conf;
-
- // FIXME(cyx): We can use std::atomic<std::shared_ptr> since c++20.
- std::shared_ptr<Aws::S3::S3Client> _client;
+ // TODO(cyx): We can use std::atomic<std::shared_ptr> since c++20.
mutable std::mutex _client_mu;
-
+ std::shared_ptr<Aws::S3::S3Client> _client;
std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> _executor;
};
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index 9142a18faf..f613b14e17 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -57,16 +57,17 @@ static const int MAX_SIZE_EACH_PART = 5 * 1024 * 1024;
static const char* STREAM_TAG = "S3FileWriter";
S3FileWriter::S3FileWriter(Path path, std::shared_ptr<Aws::S3::S3Client> client,
- const S3Conf& s3_conf)
- : FileWriter(std::move(path)), _client(client), _s3_conf(s3_conf) {
+ const S3Conf& s3_conf, FileSystemSPtr fs)
+ : FileWriter(std::move(path), fs), _client(client), _s3_conf(s3_conf) {
DorisMetrics::instance()->s3_file_open_writing->increment(1);
DorisMetrics::instance()->s3_file_writer_total->increment(1);
}
S3FileWriter::~S3FileWriter() {
- if (!_closed) {
- WARN_IF_ERROR(abort(), fmt::format("Cannot abort {}", _path.native()));
+ if (_opened) {
+ close();
}
+ CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed;
}
Status S3FileWriter::close() {
@@ -91,34 +92,6 @@ Status S3FileWriter::abort() {
outcome.GetError().GetMessage());
}
-Status S3FileWriter::_open() {
- CreateMultipartUploadRequest create_request;
- create_request.WithBucket(_s3_conf.bucket).WithKey(_path.native());
- create_request.SetContentType("text/plain");
-
- _reset_stream();
- auto outcome = _client->CreateMultipartUpload(create_request);
-
- if (outcome.IsSuccess()) {
- _upload_id = outcome.GetResult().GetUploadId();
- LOG(INFO) << "create multi part upload successfully (endpoint=" << _s3_conf.endpoint
- << ", bucket=" << _s3_conf.bucket << ", key=" << _path.native()
- << ") upload_id: " << _upload_id;
- return Status::OK();
- }
- return Status::IOError(
- "failed to create multi part upload (endpoint={}, bucket={}, key={}): {}",
- _s3_conf.endpoint, _s3_conf.bucket, _path.native(), outcome.GetError().GetMessage());
-}
-
-Status S3FileWriter::append(const Slice& data) {
- Status st = appendv(&data, 1);
- if (st.ok()) {
- DorisMetrics::instance()->s3_bytes_written_total->increment(data.size);
- }
- return st;
-}
-
Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) {
DCHECK(!_closed);
if (!_is_open) {
@@ -140,6 +113,34 @@ Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) {
return Status::OK();
}
+Status S3FileWriter::finalize() {
+ DCHECK(!_closed);
+ if (_opened) {
+ _close();
+ }
+ return Status::OK();
+}
+
+Status S3FileWriter::_open() {
+ CreateMultipartUploadRequest create_request;
+ create_request.WithBucket(_s3_conf.bucket).WithKey(_path.native());
+ create_request.SetContentType("text/plain");
+
+ _reset_stream();
+ auto outcome = _client->CreateMultipartUpload(create_request);
+
+ if (outcome.IsSuccess()) {
+ _upload_id = outcome.GetResult().GetUploadId();
+ LOG(INFO) << "create multi part upload successfully (endpoint=" << _s3_conf.endpoint
+ << ", bucket=" << _s3_conf.bucket << ", key=" << _path.native()
+ << ") upload_id: " << _upload_id;
+ return Status::OK();
+ }
+ return Status::IOError(
+ "failed to create multi part upload when open (endpoint={}, bucket={}, key={}): {}",
+ _s3_conf.endpoint, _s3_conf.bucket, _path.native(), outcome.GetError().GetMessage());
+}
+
Status S3FileWriter::_upload_part() {
if (_stream_ptr->str().size() == 0) {
return Status::OK();
@@ -192,14 +193,6 @@ void S3FileWriter::_reset_stream() {
_stream_ptr = Aws::MakeShared<Aws::StringStream>(STREAM_TAG, "");
}
-Status S3FileWriter::finalize() {
- DCHECK(!_closed);
- if (_is_open) {
- _close();
- }
- return Status::OK();
-}
-
Status S3FileWriter::_close() {
if (_closed) {
return Status::OK();
@@ -223,7 +216,8 @@ Status S3FileWriter::_close() {
if (!compute_outcome.IsSuccess()) {
return Status::IOError(
- "failed to create multi part upload (endpoint={}, bucket={}, key={}): {}",
+ "failed to create multi part upload when close (endpoint={}, bucket={}, "
+ "key={}): {}",
_s3_conf.endpoint, _s3_conf.bucket, _path.native(),
compute_outcome.GetError().GetMessage());
}
@@ -241,9 +235,5 @@ Status S3FileWriter::_close() {
return Status::OK();
}
-Status S3FileWriter::write_at(size_t offset, const Slice& data) {
- return Status::NotSupported("not support");
-}
-
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h
index 12dd32de9f..87aa58680f 100644
--- a/be/src/io/fs/s3_file_writer.h
+++ b/be/src/io/fs/s3_file_writer.h
@@ -36,35 +36,22 @@ namespace io {
class S3FileWriter final : public FileWriter {
public:
- S3FileWriter(Path path, std::shared_ptr<Aws::S3::S3Client> client, const S3Conf& s3_conf);
+ S3FileWriter(Path path, std::shared_ptr<Aws::S3::S3Client> client, const S3Conf& s3_conf,
+ FileSystemSPtr fs);
~S3FileWriter() override;
Status close() override;
-
Status abort() override;
-
- Status append(const Slice& data) override;
-
Status appendv(const Slice* data, size_t data_cnt) override;
-
- Status write_at(size_t offset, const Slice& data) override;
-
Status finalize() override;
-
- size_t bytes_appended() const override { return _bytes_appended; }
-
- FileSystemSPtr fs() const override { return _fs; }
-
-private:
- std::shared_ptr<S3FileSystem> _fs;
+ Status write_at(size_t offset, const Slice& data) override {
+ return Status::NotSupported("not support");
+ }
private:
Status _close();
-
Status _open();
-
Status _upload_part();
-
void _reset_stream();
private:
@@ -73,7 +60,6 @@ private:
std::string _upload_id;
bool _is_open = false;
bool _closed = false;
- size_t _bytes_appended = 0;
std::shared_ptr<Aws::StringStream> _stream_ptr;
// Current Part Num for CompletedPart
diff --git a/be/src/io/fs/stream_load_pipe.cpp b/be/src/io/fs/stream_load_pipe.cpp
index 9d33556bd2..f0e1be2c6e 100644
--- a/be/src/io/fs/stream_load_pipe.cpp
+++ b/be/src/io/fs/stream_load_pipe.cpp
@@ -39,8 +39,8 @@ StreamLoadPipe::~StreamLoadPipe() {
}
}
-Status StreamLoadPipe::read_at(size_t /*offset*/, Slice result, const IOContext& /*io_ctx*/,
- size_t* bytes_read) {
+Status StreamLoadPipe::read_at_impl(size_t /*offset*/, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) {
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->orphan_mem_tracker());
*bytes_read = 0;
size_t bytes_req = result.size;
@@ -98,8 +98,7 @@ Status StreamLoadPipe::read_one_message(std::unique_ptr<uint8_t[]>* data, size_t
// _total_length > 0, read the entire data
data->reset(new uint8_t[_total_length]);
Slice result(data->get(), _total_length);
- IOContext io_ctx;
- Status st = read_at(0, result, io_ctx, length);
+ Status st = read_at(0, result, length);
return st;
}
diff --git a/be/src/io/fs/stream_load_pipe.h b/be/src/io/fs/stream_load_pipe.h
index 75401b8c20..33e72c57a1 100644
--- a/be/src/io/fs/stream_load_pipe.h
+++ b/be/src/io/fs/stream_load_pipe.h
@@ -47,9 +47,6 @@ public:
Status append(const ByteBufferPtr& buf) override;
- Status read_at(size_t offset, Slice result, const IOContext& io_ctx,
- size_t* bytes_read) override;
-
const Path& path() const override { return _path; }
size_t size() const override { return 0; }
@@ -72,6 +69,10 @@ public:
FileSystemSPtr fs() const override { return nullptr; }
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
private:
// read the next buffer from _buf_queue
Status _read_next_buffer(std::unique_ptr<uint8_t[]>* data, size_t* length);
diff --git a/be/src/io/hdfs_writer.cpp b/be/src/io/hdfs_writer.cpp
deleted file mode 100644
index e6814438f1..0000000000
--- a/be/src/io/hdfs_writer.cpp
+++ /dev/null
@@ -1,154 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "io/hdfs_writer.h"
-
-#include <filesystem>
-
-#include "common/logging.h"
-#include "service/backend_options.h"
-
-namespace doris {
-
-HDFSWriter::HDFSWriter(const std::map<std::string, std::string>& properties,
- const std::string& path)
- : _properties(properties), _path(path), _hdfs_fs(nullptr) {
- _parse_properties(_properties);
-}
-
-HDFSWriter::~HDFSWriter() {
- close();
-}
-
-Status HDFSWriter::open() {
- if (_namenode.empty()) {
- LOG(WARNING) << "hdfs properties is incorrect.";
- return Status::InternalError("hdfs properties is incorrect");
- }
- // if the format of _path is hdfs://ip:port/path, replace it to /path.
- // path like hdfs://ip:port/path can't be used by libhdfs3.
- if (_path.find(_namenode) != _path.npos) {
- _path = _path.substr(_namenode.size());
- }
- RETURN_IF_ERROR(_connect());
- if (_hdfs_fs == nullptr) {
- return Status::InternalError("HDFS writer open without client");
- }
- int exists = hdfsExists(_hdfs_fs, _path.c_str());
- if (exists == 0) {
- // the path already exists
- return Status::AlreadyExist("{} already exists.", _path);
- }
-
- std::filesystem::path hdfs_path(_path);
- std::string hdfs_dir = hdfs_path.parent_path().string();
- exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str());
- if (exists != 0) {
- VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir;
- int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str());
- if (ret != 0) {
- std::stringstream ss;
- ss << "create dir failed. "
- << "(BE: " << BackendOptions::get_localhost() << ")"
- << " namenode: " << _namenode << " path: " << hdfs_dir
- << ", err: " << hdfsGetLastError();
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- }
- // open file
- _hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0);
- if (_hdfs_file == nullptr) {
- std::stringstream ss;
- ss << "open file failed. "
- << "(BE: " << BackendOptions::get_localhost() << ")"
- << " namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError();
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- VLOG_NOTICE << "open file. namenode:" << _namenode << ", path:" << _path;
- return Status::OK();
-}
-
-Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len) {
- if (buf_len == 0) {
- *written_len = 0;
- return Status::OK();
- }
- int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len);
- if (result < 0) {
- std::stringstream ss;
- ss << "write file failed. "
- << "(BE: " << BackendOptions::get_localhost() << ")"
- << "namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError();
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
-
- *written_len = (unsigned int)result;
- return Status::OK();
-}
-
-Status HDFSWriter::close() {
- if (_closed) {
- return Status::OK();
- }
- _closed = true;
- if (_hdfs_fs == nullptr) {
- return Status::OK();
- }
- if (_hdfs_file == nullptr) {
- // Even if there is an error, the resources associated with the hdfsFS will be freed.
- hdfsDisconnect(_hdfs_fs);
- return Status::OK();
- }
- int result = hdfsFlush(_hdfs_fs, _hdfs_file);
- if (result == -1) {
- std::stringstream ss;
- ss << "failed to flush hdfs file. "
- << "(BE: " << BackendOptions::get_localhost() << ")"
- << "namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError();
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- hdfsCloseFile(_hdfs_fs, _hdfs_file);
- hdfsDisconnect(_hdfs_fs);
-
- _hdfs_file = nullptr;
- _hdfs_fs = nullptr;
- return Status::OK();
-}
-
-Status HDFSWriter::_connect() {
- HDFSCommonBuilder builder;
- RETURN_IF_ERROR(createHDFSBuilder(_properties, &builder));
- _hdfs_fs = hdfsBuilderConnect(builder.get());
- if (_hdfs_fs == nullptr) {
- return Status::InternalError("connect to hdfs failed. namenode address:{}, error {}",
- _namenode, hdfsGetLastError());
- }
- return Status::OK();
-}
-
-void HDFSWriter::_parse_properties(const std::map<std::string, std::string>& prop) {
- auto iter = prop.find(FS_KEY);
- if (iter != prop.end()) {
- _namenode = iter->second;
- }
-}
-
-} // end namespace doris
diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h
new file mode 100644
index 0000000000..3e5bce3fc0
--- /dev/null
+++ b/be/src/io/io_common.h
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "gen_cpp/Types_types.h"
+
+namespace doris {
+
+enum ReaderType {
+ READER_QUERY = 0,
+ READER_ALTER_TABLE = 1,
+ READER_BASE_COMPACTION = 2,
+ READER_CUMULATIVE_COMPACTION = 3,
+ READER_CHECKSUM = 4,
+ READER_COLD_DATA_COMPACTION = 5,
+ READER_SEGMENT_COMPACTION = 6,
+};
+
+namespace io {
+
+struct FileCacheStatistics {
+ int64_t num_io_total = 0;
+ int64_t num_io_hit_cache = 0;
+ int64_t num_io_bytes_read_total = 0;
+ int64_t num_io_bytes_read_from_file_cache = 0;
+ int64_t num_io_bytes_read_from_write_cache = 0;
+ int64_t num_io_written_in_file_cache = 0;
+ int64_t num_io_bytes_written_in_file_cache = 0;
+ int64_t num_io_bytes_skip_cache = 0;
+};
+
+class IOContext {
+public:
+ IOContext() = default;
+
+ IOContext(const TUniqueId* query_id_, FileCacheStatistics* stats_, bool is_presistent_,
+ bool use_disposable_cache_, bool read_segment_index_, bool enable_file_cache)
+ : query_id(query_id_),
+ is_persistent(is_presistent_),
+ use_disposable_cache(use_disposable_cache_),
+ read_segment_index(read_segment_index_),
+ file_cache_stats(stats_),
+ enable_file_cache(enable_file_cache) {}
+ ReaderType reader_type;
+ const TUniqueId* query_id = nullptr;
+ bool is_persistent = false;
+ bool use_disposable_cache = false;
+ bool read_segment_index = false;
+ FileCacheStatistics* file_cache_stats = nullptr;
+ bool enable_file_cache = true;
+};
+
+} // namespace io
+} // namespace doris
diff --git a/be/src/io/local_file_writer.cpp b/be/src/io/local_file_writer.cpp
deleted file mode 100644
index b6c6dfcefa..0000000000
--- a/be/src/io/local_file_writer.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "io/local_file_writer.h"
-
-#include "service/backend_options.h"
-#include "util/error_util.h"
-#include "util/file_utils.h"
-
-namespace doris {
-
-LocalFileWriter::LocalFileWriter(const std::string& path, int64_t start_offset)
- : _path(path), _start_offset(start_offset), _fp(nullptr) {}
-
-LocalFileWriter::~LocalFileWriter() {
- close();
-}
-
-Status LocalFileWriter::open() {
- RETURN_IF_ERROR(_check_file_path(_path));
-
- _fp = fopen(_path.c_str(), "w+");
- if (_fp == nullptr) {
- return Status::InternalError("Open file failed. path={}, errno= {}, description={}", _path,
- errno, get_str_err_msg());
- }
-
- if (_start_offset != 0) {
- int success = fseek(_fp, _start_offset, SEEK_SET);
- if (success != 0) {
- return Status::InternalError(
- "Seek to start_offset failed. offset={}, errno= {}, description={}",
- _start_offset, errno, get_str_err_msg());
- }
- }
-
- return Status::OK();
-}
-
-Status LocalFileWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len) {
- size_t bytes_written = fwrite(buf, 1, buf_len, _fp);
- if (bytes_written < buf_len) {
- return Status::InternalError(
- "fail to write to file. len={}, path={}, failed with errno={}, description={}",
- buf_len, _path, errno, get_str_err_msg());
- }
-
- *written_len = bytes_written;
- return Status::OK();
-}
-
-Status LocalFileWriter::close() {
- if (_fp != nullptr) {
- fclose(_fp);
- _fp = nullptr;
- }
- return Status::OK();
-}
-
-Status LocalFileWriter::_check_file_path(const std::string& file_path) {
- // For local file writer, the file_path is a local dir.
- // Here we do a simple security verification by checking whether the file exists.
- // Because the file path is currently arbitrarily specified by the user,
- // Doris is not responsible for ensuring the correctness of the path.
- // This is just to prevent overwriting the existing file.
- if (FileUtils::check_exist(file_path)) {
- return Status::InternalError("File already exists: {}. Host: {}", file_path,
- BackendOptions::get_localhost());
- }
-
- return Status::OK();
-}
-
-} // end namespace doris
diff --git a/be/src/io/local_file_writer.h b/be/src/io/local_file_writer.h
deleted file mode 100644
index 48ba05764b..0000000000
--- a/be/src/io/local_file_writer.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <stdio.h>
-
-#include "io/file_writer.h"
-
-namespace doris {
-
-class RuntimeState;
-
-class LocalFileWriter : public FileWriter {
-public:
- LocalFileWriter(const std::string& path, int64_t start_offset);
-
- ~LocalFileWriter() override;
-
- Status open() override;
-
- virtual Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override;
-
- virtual Status close() override;
-
-private:
- static Status _check_file_path(const std::string& file_path);
-
- std::string _path;
- int64_t _start_offset;
- FILE* _fp;
-};
-
-} // end namespace doris
diff --git a/be/src/io/s3_writer.cpp b/be/src/io/s3_writer.cpp
deleted file mode 100644
index ac9bbf997e..0000000000
--- a/be/src/io/s3_writer.cpp
+++ /dev/null
@@ -1,138 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "io/s3_writer.h"
-
-#include <aws/core/utils/FileSystemUtils.h>
-#include <aws/s3/S3Client.h>
-#include <aws/s3/model/HeadObjectRequest.h>
-#include <aws/s3/model/PutObjectRequest.h>
-
-#include "runtime/exec_env.h"
-#include "runtime/tmp_file_mgr.h"
-#include "service/backend_options.h"
-#include "util/s3_uri.h"
-#include "util/s3_util.h"
-
-namespace doris {
-
-#ifndef CHECK_S3_CLIENT
-#define CHECK_S3_CLIENT(client) \
- if (!client) { \
- return Status::InternalError("init aws s3 client error."); \
- }
-#endif
-
-S3Writer::S3Writer(const std::map<std::string, std::string>& properties, const std::string& path,
- int64_t start_offset)
- : _properties(properties),
- _path(path),
- _uri(path),
- _client(ClientFactory::instance().create(_properties)) {
- std::string tmp_path = ExecEnv::GetInstance()->tmp_file_mgr()->get_tmp_dir_path();
- LOG(INFO) << "init aws s3 client with tmp path " << tmp_path;
- if (tmp_path.at(tmp_path.size() - 1) != '/') {
- tmp_path.append("/");
- }
- _temp_file = std::make_shared<Aws::Utils::TempFile>(
- tmp_path.c_str(), ".doris_tmp",
- std::ios_base::binary | std::ios_base::trunc | std::ios_base::in | std::ios_base::out);
- DCHECK(_client) << "init aws s3 client error.";
-}
-
-S3Writer::~S3Writer() {
- close();
-}
-
-Status S3Writer::open() {
- CHECK_S3_CLIENT(_client);
- if (!_uri.parse()) {
- return Status::InvalidArgument("s3 uri is invalid: {}", _path);
- }
- Aws::S3::Model::HeadObjectRequest request;
- request.WithBucket(_uri.get_bucket()).WithKey(_uri.get_key());
- Aws::S3::Model::HeadObjectOutcome response = _client->HeadObject(request);
- if (response.IsSuccess()) {
- return Status::AlreadyExist("{} already exists.", _path);
- } else if (response.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) {
- return Status::OK();
- } else {
- return Status::InternalError("Error: [{}:{}] at {}", response.GetError().GetExceptionName(),
- response.GetError().GetMessage(),
- BackendOptions::get_localhost());
- }
-}
-
-Status S3Writer::write(const uint8_t* buf, size_t buf_len, size_t* written_len) {
- if (buf_len == 0) {
- *written_len = 0;
- return Status::OK();
- }
- if (!_temp_file) {
- RETURN_NOT_OK_STATUS_WITH_WARN(
- Status::BufferAllocFailed(
- "The internal temporary file is not writable for {}. at {}",
- strerror(errno), BackendOptions::get_localhost()),
- "write temp file error");
- }
- _temp_file->write(reinterpret_cast<const char*>(buf), buf_len);
- if (!_temp_file->good()) {
- RETURN_NOT_OK_STATUS_WITH_WARN(
- Status::BufferAllocFailed(
- "Could not append to the internal temporary file for {}. at {}",
- strerror(errno), BackendOptions::get_localhost()),
- "write temp file error");
- }
- *written_len = buf_len;
- return Status::OK();
-}
-
-Status S3Writer::close() {
- if (_temp_file) {
- RETURN_IF_ERROR(_sync());
- _temp_file.reset();
- }
- return Status::OK();
-}
-
-Status S3Writer::_sync() {
- if (!_temp_file) {
- RETURN_NOT_OK_STATUS_WITH_WARN(
- Status::BufferAllocFailed(
- "The internal temporary file is not writable for {}. at {}",
- strerror(errno), BackendOptions::get_localhost()),
- "write temp file error");
- }
- CHECK_S3_CLIENT(_client);
- Aws::S3::Model::PutObjectRequest request;
- request.WithBucket(_uri.get_bucket()).WithKey(_uri.get_key());
- long offset = _temp_file->tellp();
- _temp_file->seekg(0);
- request.SetBody(_temp_file);
- request.SetContentLength(offset);
- auto response = _client->PutObject(request);
- _temp_file->clear();
- _temp_file->seekp(offset);
- if (response.IsSuccess()) {
- return Status::OK();
- } else {
- return Status::InternalError("Error: [{}:{}] at {}", response.GetError().GetExceptionName(),
- response.GetError().GetMessage(),
- BackendOptions::get_localhost());
- }
-}
-} // end namespace doris
diff --git a/be/src/io/s3_writer.h b/be/src/io/s3_writer.h
deleted file mode 100644
index a0ad9954a9..0000000000
--- a/be/src/io/s3_writer.h
+++ /dev/null
@@ -1,59 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <map>
-#include <string>
-
-#include "io/file_writer.h"
-#include "util/s3_uri.h"
-
-namespace Aws {
-namespace Utils {
-class TempFile;
-}
-namespace S3 {
-class S3Client;
-}
-} // namespace Aws
-
-namespace doris {
-class S3Writer : public FileWriter {
-public:
- S3Writer(const std::map<std::string, std::string>& properties, const std::string& path,
- int64_t start_offset);
- ~S3Writer();
- Status open() override;
-
- // Writes up to count bytes from the buffer pointed buf to the file.
- // NOTE: the number of bytes written may be less than count if.
- Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override;
-
- Status close() override;
-
-private:
- Status _sync();
-
- const std::map<std::string, std::string>& _properties;
- std::string _path;
- S3URI _uri;
- std::shared_ptr<Aws::S3::S3Client> _client;
- std::shared_ptr<Aws::Utils::TempFile> _temp_file;
-};
-
-} // end namespace doris
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index c37bf8938f..067cab7bc9 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -35,7 +35,6 @@ add_library(Olap STATIC
cumulative_compaction_policy.cpp
delete_handler.cpp
delta_writer.cpp
- file_helper.cpp
hll.cpp
inverted_index_parser.cpp
itoken_extractor.cpp
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 6bd647a3be..8fd7d357ef 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -44,7 +44,6 @@
#include "gutil/strings/substitute.h"
#include "io/fs/local_file_system.h"
#include "io/fs/path.h"
-#include "olap/file_helper.h"
#include "olap/olap_define.h"
#include "olap/rowset/beta_rowset.h"
#include "olap/rowset/rowset_meta_manager.h"
diff --git a/be/src/olap/file_header.h b/be/src/olap/file_header.h
new file mode 100644
index 0000000000..1732331a0d
--- /dev/null
+++ b/be/src/olap/file_header.h
@@ -0,0 +1,240 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <stdio.h>
+#include <sys/stat.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "io/fs/file_reader.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/local_file_system.h"
+#include "olap/lru_cache.h"
+#include "olap/olap_common.h"
+#include "olap/olap_define.h"
+#include "olap/utils.h"
+#include "util/debug_util.h"
+
+namespace doris {
+
+typedef struct _FixedFileHeader {
+ // the length of the entire file
+ uint32_t file_length;
+ // Checksum of the file's contents except the FileHeader
+ uint32_t checksum;
+ // Protobuf length of section
+ uint32_t protobuf_length;
+ // Checksum of Protobuf part
+ uint32_t protobuf_checksum;
+} __attribute__((packed)) FixedFileHeader;
+
+typedef struct _FixedFileHeaderV2 {
+ uint64_t magic_number;
+ uint32_t version;
+ // the length of the entire file
+ uint64_t file_length;
+ // Checksum of the file's contents except the FileHeader
+ uint32_t checksum;
+ // Protobuf length of section
+ uint64_t protobuf_length;
+ // Checksum of Protobuf part
+ uint32_t protobuf_checksum;
+} __attribute__((packed)) FixedFileHeaderV2;
+
+template <typename MessageType, typename ExtraType = uint32_t>
+class FileHeader {
+public:
+ FileHeader(const std::string& file_path) : _file_path(file_path) {
+ memset(&_fixed_file_header, 0, sizeof(_fixed_file_header));
+ memset(&_extra_fixed_header, 0, sizeof(_extra_fixed_header));
+ _fixed_file_header_size = sizeof(_fixed_file_header);
+ }
+ ~FileHeader() {}
+
+ // To calculate the length of the proto part, it needs to be called after the proto is operated,
+ // and prepare must be called before calling serialize
+ Status prepare();
+
+ // call prepare() first, serialize() will write fixed header and protobuffer.
+ // Write the header to the starting position of the incoming file handle
+ Status serialize();
+
+ // read from file, validate file length, signature and alder32 of protobuffer.
+ // Read the header from the beginning of the incoming file handle
+ Status deserialize();
+
+ // Check the validity of Header
+ // it is actually call deserialize().
+ Status validate();
+
+ uint64_t file_length() const { return _fixed_file_header.file_length; }
+ uint32_t checksum() const { return _fixed_file_header.checksum; }
+ const ExtraType& extra() const { return _extra_fixed_header; }
+ ExtraType* mutable_extra() { return &_extra_fixed_header; }
+ const MessageType& message() const { return _proto; }
+ MessageType* mutable_message() { return &_proto; }
+ uint64_t size() const {
+ return _fixed_file_header_size + sizeof(_extra_fixed_header) +
+ _fixed_file_header.protobuf_length;
+ }
+
+ void set_file_length(uint64_t file_length) { _fixed_file_header.file_length = file_length; }
+ void set_checksum(uint32_t checksum) { _fixed_file_header.checksum = checksum; }
+
+private:
+ std::string _file_path;
+ FixedFileHeaderV2 _fixed_file_header;
+ uint32_t _fixed_file_header_size;
+
+ std::string _proto_string;
+ ExtraType _extra_fixed_header;
+ MessageType _proto;
+};
+
+// FileHeader implementation
+template <typename MessageType, typename ExtraType>
+Status FileHeader<MessageType, ExtraType>::prepare() {
+ try {
+ if (!_proto.SerializeToString(&_proto_string)) {
+ LOG(WARNING) << "serialize file header to string error. [path='" << _file_path << "']";
+ return Status::Error<ErrorCode::SERIALIZE_PROTOBUF_ERROR>();
+ }
+ } catch (...) {
+ LOG(WARNING) << "serialize file header to string error. [path='" << _file_path << "']";
+ return Status::Error<ErrorCode::SERIALIZE_PROTOBUF_ERROR>();
+ }
+
+ _fixed_file_header.protobuf_checksum =
+ olap_adler32(olap_adler32_init(), _proto_string.c_str(), _proto_string.size());
+
+ _fixed_file_header.checksum = 0;
+ _fixed_file_header.protobuf_length = _proto_string.size();
+ _fixed_file_header.file_length = size();
+ _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED;
+ _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER;
+
+ return Status::OK();
+}
+
+template <typename MessageType, typename ExtraType>
+Status FileHeader<MessageType, ExtraType>::serialize() {
+ // write to file
+ io::FileWriterPtr file_writer;
+ RETURN_IF_ERROR(io::global_local_filesystem()->create_file(_file_path, &file_writer));
+ RETURN_IF_ERROR(file_writer->write_at(
+ 0, {(const uint8_t*)&_fixed_file_header, _fixed_file_header_size}));
+ RETURN_IF_ERROR(file_writer->write_at(
+ _fixed_file_header_size,
+ {(const uint8_t*)&_extra_fixed_header, sizeof(_extra_fixed_header)}));
+ RETURN_IF_ERROR(file_writer->write_at(_fixed_file_header_size + sizeof(_extra_fixed_header),
+ {_proto_string}));
+ return file_writer->close();
+}
+
+template <typename MessageType, typename ExtraType>
+Status FileHeader<MessageType, ExtraType>::deserialize() {
+ io::FileReaderSPtr file_reader;
+ RETURN_IF_ERROR(io::global_local_filesystem()->open_file(_file_path, &file_reader));
+ off_t real_file_length = 0;
+ uint32_t real_protobuf_checksum = 0;
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(file_reader->read_at(
+ 0, {(const uint8_t*)&_fixed_file_header, _fixed_file_header_size}, &bytes_read));
+
+ //Status read_at(size_t offset, Slice result, size_t* bytes_read,
+ // const IOContext* io_ctx = nullptr);
+
+ if (_fixed_file_header.magic_number != OLAP_FIX_HEADER_MAGIC_NUMBER) {
+ VLOG_TRACE << "old fix header found, magic num=" << _fixed_file_header.magic_number;
+ FixedFileHeader tmp_header;
+ RETURN_IF_ERROR(file_reader->read_at(0, {(const uint8_t*)&tmp_header, sizeof(tmp_header)},
+ &bytes_read));
+ _fixed_file_header.file_length = tmp_header.file_length;
+ _fixed_file_header.checksum = tmp_header.checksum;
+ _fixed_file_header.protobuf_length = tmp_header.protobuf_length;
+ _fixed_file_header.protobuf_checksum = tmp_header.protobuf_checksum;
+ _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER;
+ _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED;
+ _fixed_file_header_size = sizeof(tmp_header);
+ }
+
+ VLOG_NOTICE << "fix head loaded. file_length=" << _fixed_file_header.file_length
+ << ", checksum=" << _fixed_file_header.checksum
+ << ", protobuf_length=" << _fixed_file_header.protobuf_length
+ << ", magic_number=" << _fixed_file_header.magic_number
+ << ", version=" << _fixed_file_header.version;
+
+ RETURN_IF_ERROR(file_reader->read_at(
+ _fixed_file_header_size,
+ {(const uint8_t*)&_extra_fixed_header, sizeof(_extra_fixed_header)}, &bytes_read));
+
+ std::unique_ptr<char[]> buf(new (std::nothrow) char[_fixed_file_header.protobuf_length]);
+ if (nullptr == buf.get()) {
+ char errmsg[64];
+ LOG(WARNING) << "malloc protobuf buf error. file=" << file_reader->path().native()
+ << ", error=" << strerror_r(errno, errmsg, 64);
+ return Status::Error<ErrorCode::MEM_ALLOC_FAILED>();
+ }
+ RETURN_IF_ERROR(file_reader->read_at(_fixed_file_header_size + sizeof(_extra_fixed_header),
+ {buf.get(), _fixed_file_header.protobuf_length},
+ &bytes_read));
+ real_file_length = file_reader->size();
+
+ if (file_length() != static_cast<uint64_t>(real_file_length)) {
+ LOG(WARNING) << "file length is not match. file=" << file_reader->path().native()
+ << ", file_length=" << file_length()
+ << ", real_file_length=" << real_file_length;
+ return Status::Error<ErrorCode::FILE_DATA_ERROR>();
+ }
+
+ // check proto checksum
+ real_protobuf_checksum =
+ olap_adler32(olap_adler32_init(), buf.get(), _fixed_file_header.protobuf_length);
+
+ if (real_protobuf_checksum != _fixed_file_header.protobuf_checksum) {
+ LOG(WARNING) << "checksum is not match. file=" << file_reader->path().native()
+ << ", expect=" << _fixed_file_header.protobuf_checksum
+ << ", actual=" << real_protobuf_checksum;
+ return Status::Error<ErrorCode::CHECKSUM_ERROR>();
+ }
+
+ try {
+ std::string protobuf_str(buf.get(), _fixed_file_header.protobuf_length);
+
+ if (!_proto.ParseFromString(protobuf_str)) {
+ LOG(WARNING) << "fail to parse file content to protobuf object. file="
+ << file_reader->path().native();
+ return Status::Error<ErrorCode::PARSE_PROTOBUF_ERROR>();
+ }
+ } catch (...) {
+ LOG(WARNING) << "fail to load protobuf. file='" << file_reader->path().native();
+ return Status::Error<ErrorCode::PARSE_PROTOBUF_ERROR>();
+ }
+
+ return Status::OK();
+}
+
+template <typename MessageType, typename ExtraType>
+Status FileHeader<MessageType, ExtraType>::validate() {
+ return deserialize();
+}
+
+} // namespace doris
diff --git a/be/src/olap/file_helper.cpp b/be/src/olap/file_helper.cpp
deleted file mode 100644
index a1779484f8..0000000000
--- a/be/src/olap/file_helper.cpp
+++ /dev/null
@@ -1,377 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/file_helper.h"
-
-#include <errno.h>
-#include <stdio.h>
-#include <sys/stat.h>
-
-#include <string>
-#include <vector>
-
-#include "common/config.h"
-#include "olap/olap_common.h"
-#include "olap/olap_define.h"
-#include "olap/storage_engine.h"
-#include "olap/utils.h"
-#include "util/debug_util.h"
-
-using std::string;
-
-namespace doris {
-using namespace ErrorCode;
-
-FileHandler::FileHandler() : _fd(-1), _wr_length(0), _file_name("") {}
-
-FileHandler::~FileHandler() {
- this->close();
-}
-
-Status FileHandler::open(const string& file_name, int flag) {
- if (_fd != -1 && _file_name == file_name) {
- return Status::OK();
- }
-
- RETURN_IF_ERROR(this->close());
-
- _fd = ::open(file_name.c_str(), flag);
-
- if (_fd < 0) {
- char errmsg[64];
- LOG(WARNING) << "failed to open file. [err=" << strerror_r(errno, errmsg, 64)
- << ", file_name='" << file_name << "' flag=" << flag << "]";
- if (errno == EEXIST) {
- return Status::Error<FILE_ALREADY_EXIST>();
- }
- return Status::Error<IO_ERROR>();
- }
-
- VLOG_NOTICE << "success to open file. file_name=" << file_name << ", mode=" << flag
- << " fd=" << _fd;
- _file_name = file_name;
- return Status::OK();
-}
-
-Status FileHandler::open_with_mode(const string& file_name, int flag, int mode) {
- if (_fd != -1 && _file_name == file_name) {
- return Status::OK();
- }
-
- RETURN_IF_ERROR(this->close());
-
- _fd = ::open(file_name.c_str(), flag, mode);
-
- if (_fd < 0) {
- char err_buf[64];
- LOG(WARNING) << "failed to open file. [err=" << strerror_r(errno, err_buf, 64)
- << " file_name='" << file_name << "' flag=" << flag << " mode=" << mode << "]";
- if (errno == EEXIST) {
- return Status::Error<FILE_ALREADY_EXIST>();
- }
- return Status::Error<IO_ERROR>();
- }
-
- VLOG_NOTICE << "success to open file. file_name=" << file_name << ", mode=" << mode
- << ", fd=" << _fd;
- _file_name = file_name;
- return Status::OK();
-}
-
-Status FileHandler::_release() {
- return Status::OK();
-}
-
-Status FileHandler::close() {
- if (_fd < 0) {
- return Status::OK();
- }
-
-#ifndef __APPLE__
- // try to sync page cache if have written some bytes
- if (_wr_length > 0) {
- posix_fadvise(_fd, 0, 0, POSIX_FADV_DONTNEED);
- // Clean dirty pages and wait for io queue empty and return
- sync_file_range(_fd, 0, 0, SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER);
- _wr_length = 0;
- }
-#endif
-
- // In some extreme cases (fd is available, but fsync fails) can cause handle leaks
- if (::close(_fd) < 0) {
- char errmsg[64];
- LOG(WARNING) << "failed to close file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << "]";
- return Status::Error<IO_ERROR>();
- }
-
- VLOG_NOTICE << "finished to close file. "
- << "file_name=" << _file_name << ", fd=" << _fd;
- _fd = -1;
- _file_name = "";
- _wr_length = 0;
- return Status::OK();
-}
-
-Status FileHandler::pread(void* buf, size_t size, size_t offset) {
- char* ptr = reinterpret_cast<char*>(buf);
-
- while (size > 0) {
- ssize_t rd_size = ::pread(_fd, ptr, size, offset);
-
- if (rd_size < 0) {
- char errmsg[64];
- LOG(WARNING) << "failed to pread from file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << " size=" << size
- << " offset=" << offset << "]";
- return Status::Error<IO_ERROR>();
- } else if (0 == rd_size) {
- char errmsg[64];
- LOG(WARNING) << "read unenough from file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << " size=" << size
- << " offset=" << offset << "]";
- return Status::Error<READ_UNENOUGH>();
- }
-
- size -= rd_size;
- offset += rd_size;
- ptr += rd_size;
- }
-
- return Status::OK();
-}
-
-Status FileHandler::write(const void* buf, size_t buf_size) {
- size_t org_buf_size = buf_size;
- const char* ptr = reinterpret_cast<const char*>(buf);
- while (buf_size > 0) {
- ssize_t wr_size = ::write(_fd, ptr, buf_size);
-
- if (wr_size < 0) {
- char errmsg[64];
- LOG(WARNING) << "failed to write to file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size
- << "]";
- return Status::Error<IO_ERROR>();
- } else if (0 == wr_size) {
- char errmsg[64];
- LOG(WARNING) << "write unenough to file. [err=" << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size
- << "]";
- return Status::Error<IO_ERROR>();
- }
-
- buf_size -= wr_size;
- ptr += wr_size;
- }
- _wr_length += org_buf_size;
-
-#ifndef __APPLE__
- // try to sync page cache if cache size is bigger than threshold
- if (_wr_length >= _cache_threshold) {
- posix_fadvise(_fd, 0, 0, POSIX_FADV_DONTNEED);
- // Clean dirty pages and wait for io queue empty and return
- sync_file_range(_fd, 0, 0, SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER);
- _wr_length = 0;
- }
-#endif
-
- return Status::OK();
-}
-
-Status FileHandler::pwrite(const void* buf, size_t buf_size, size_t offset) {
- const char* ptr = reinterpret_cast<const char*>(buf);
-
- size_t org_buf_size = buf_size;
- while (buf_size > 0) {
- ssize_t wr_size = ::pwrite(_fd, ptr, buf_size, offset);
-
- if (wr_size < 0) {
- char errmsg[64];
- LOG(WARNING) << "failed to pwrite to file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size
- << " offset=" << offset << "]";
- return Status::Error<IO_ERROR>();
- } else if (0 == wr_size) {
- char errmsg[64];
- LOG(WARNING) << "pwrite unenough to file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size
- << " offset=" << offset << "]";
- return Status::Error<IO_ERROR>();
- }
-
- buf_size -= wr_size;
- ptr += wr_size;
- offset += wr_size;
- }
- _wr_length += org_buf_size;
-
- return Status::OK();
-}
-
-off_t FileHandler::length() const {
- struct stat stat_data;
-
- if (fstat(_fd, &stat_data) < 0) {
- LOG(WARNING) << "fstat error. [fd=" << _fd << "]";
- return -1;
- }
-
- return stat_data.st_size;
-}
-
-FileHandlerWithBuf::FileHandlerWithBuf() : _fp(nullptr), _file_name("") {}
-
-FileHandlerWithBuf::~FileHandlerWithBuf() {
- this->close();
-}
-
-Status FileHandlerWithBuf::open(const string& file_name, const char* mode) {
- if (_fp != nullptr && _file_name == file_name) {
- return Status::OK();
- }
-
- RETURN_IF_ERROR(this->close());
-
- _fp = ::fopen(file_name.c_str(), mode);
-
- if (nullptr == _fp) {
- char errmsg[64];
- LOG(WARNING) << "failed to open file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << file_name << "' flag='" << mode << "']";
- if (errno == EEXIST) {
- return Status::Error<FILE_ALREADY_EXIST>();
- }
- return Status::Error<IO_ERROR>();
- }
-
- VLOG_NOTICE << "success to open file. "
- << "file_name=" << file_name << ", mode=" << mode;
- _file_name = file_name;
- return Status::OK();
-}
-
-Status FileHandlerWithBuf::open_with_mode(const string& file_name, const char* mode) {
- return this->open(file_name, mode);
-}
-
-Status FileHandlerWithBuf::close() {
- if (nullptr == _fp) {
- return Status::OK();
- }
-
- // In some cases (fd is available, but fsync fails) can cause handle leaks
- if (::fclose(_fp) != 0) {
- char errmsg[64];
- LOG(WARNING) << "failed to close file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "']";
- return Status::Error<IO_ERROR>();
- }
-
- _fp = nullptr;
- _file_name = "";
- return Status::OK();
-}
-
-Status FileHandlerWithBuf::read(void* buf, size_t size) {
- if (OLAP_UNLIKELY(nullptr == _fp)) {
- LOG(WARNING) << "Fail to write, fp is nullptr!";
- return Status::Error<UNINITIALIZED>();
- }
-
- size_t rd_size = ::fread(buf, 1, size, _fp);
-
- if (rd_size == size) {
- return Status::OK();
- } else if (::feof(_fp)) {
- char errmsg[64];
- LOG(WARNING) << "read unenough from file. [err=" << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' size=" << size << " rd_size=" << rd_size
- << "]";
- return Status::Error<READ_UNENOUGH>();
- } else {
- char errmsg[64];
- LOG(WARNING) << "failed to read from file. [err=" << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' size=" << size << "]";
- return Status::Error<IO_ERROR>();
- }
-}
-
-Status FileHandlerWithBuf::pread(void* buf, size_t size, size_t offset) {
- if (OLAP_UNLIKELY(nullptr == _fp)) {
- LOG(WARNING) << "Fail to write, fp is nullptr!";
- return Status::Error<UNINITIALIZED>();
- }
-
- if (0 != ::fseek(_fp, offset, SEEK_SET)) {
- char errmsg[64];
- LOG(WARNING) << "failed to seek file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' size=" << size << " offset=" << offset
- << "]";
- return Status::Error<IO_ERROR>();
- }
-
- return this->read(buf, size);
-}
-
-Status FileHandlerWithBuf::write(const void* buf, size_t buf_size) {
- if (OLAP_UNLIKELY(nullptr == _fp)) {
- LOG(WARNING) << "Fail to write, fp is nullptr!";
- return Status::Error<UNINITIALIZED>();
- }
-
- size_t wr_size = ::fwrite(buf, 1, buf_size, _fp);
-
- if (wr_size != buf_size) {
- char errmsg[64];
- LOG(WARNING) << "failed to write to file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' size=" << buf_size << "]";
- return Status::Error<IO_ERROR>();
- }
-
- return Status::OK();
-}
-
-Status FileHandlerWithBuf::pwrite(const void* buf, size_t buf_size, size_t offset) {
- if (OLAP_UNLIKELY(nullptr == _fp)) {
- LOG(WARNING) << "Fail to write, fp is nullptr!";
- return Status::Error<UNINITIALIZED>();
- }
-
- if (0 != ::fseek(_fp, offset, SEEK_SET)) {
- char errmsg[64];
- LOG(WARNING) << "failed to seek file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' size=" << buf_size
- << " offset=" << offset << "]";
- return Status::Error<IO_ERROR>();
- }
-
- return this->write(buf, buf_size);
-}
-
-off_t FileHandlerWithBuf::length() const {
- struct stat stat_data;
-
- if (stat(_file_name.c_str(), &stat_data) < 0) {
- LOG(WARNING) << "fstat error. [file_name='" << _file_name << "']";
- return -1;
- }
-
- return stat_data.st_size;
-}
-
-} // namespace doris
diff --git a/be/src/olap/file_helper.h b/be/src/olap/file_helper.h
deleted file mode 100644
index 5b5a292968..0000000000
--- a/be/src/olap/file_helper.h
+++ /dev/null
@@ -1,430 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <stdio.h>
-#include <sys/stat.h>
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "olap/lru_cache.h"
-#include "olap/olap_common.h"
-#include "olap/olap_define.h"
-#include "olap/utils.h"
-#include "util/debug_util.h"
-
-namespace doris {
-
-typedef struct FileDescriptor {
- int fd;
- FileDescriptor(int fd) : fd(fd) {}
- ~FileDescriptor() { ::close(fd); }
-} FileDescriptor;
-
-class FileHandler {
-public:
- FileHandler();
- ~FileHandler();
-
- Status open(const std::string& file_name, int flag);
- // The argument mode specifies the permissions to use in case a new file is created.
- Status open_with_mode(const std::string& file_name, int flag, int mode);
- Status close();
-
- Status pread(void* buf, size_t size, size_t offset);
- Status write(const void* buf, size_t buf_size);
- Status pwrite(const void* buf, size_t buf_size, size_t offset);
-
- int32_t sync() { return 0; }
-
- off_t tell() const {
- off_t res = -1;
-
- if (-1 == (res = lseek(_fd, 0, SEEK_CUR))) {
- char errmsg[64];
- LOG(WARNING) << "fail to tell file. [err=" << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' fd=" << _fd << "]";
- }
-
- return res;
- }
-
- off_t length() const;
-
- off_t seek(off_t offset, int whence) {
- off_t res = -1;
-
- if (-1 == (res = lseek(_fd, offset, whence))) {
- char errmsg[64];
- LOG(WARNING) << "fail to seek file. [err=" << strerror_r(errno, errmsg, 64)
- << "file_name='" << _file_name << "' fd=" << _fd << " offset=" << offset
- << " whence=" << whence << "]";
- }
-
- return res;
- }
-
- const std::string& file_name() { return _file_name; }
-
- int fd() { return _fd; }
-
- static void _delete_cache_file_descriptor(const CacheKey& key, void* value) {
- FileDescriptor* file_desc = reinterpret_cast<FileDescriptor*>(value);
- SAFE_DELETE(file_desc);
- }
-
-private:
- Status _release();
-
- int _fd;
- off_t _wr_length;
- std::string _file_name;
-
-#ifndef __APPLE__
- const int64_t _cache_threshold = 1 << 19;
-#endif
-};
-
-class FileHandlerWithBuf {
-public:
- FileHandlerWithBuf();
- ~FileHandlerWithBuf();
-
- Status open(const std::string& file_name, const char* mode);
- // The argument mode specifies the permissions to use in case a new file is created.
- Status open_with_mode(const std::string& file_name, const char* mode);
- Status close();
-
- Status read(void* buf, size_t size);
- Status pread(void* buf, size_t size, size_t offset);
- Status write(const void* buf, size_t buf_size);
- Status pwrite(const void* buf, size_t buf_size, size_t offset);
-
- int32_t sync() {
- int32_t res = -1;
- if (0 != (res = ::fflush(_fp))) {
- char errmsg[64];
- LOG(WARNING) << "fail to fsync file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "']";
- }
- return res;
- }
-
- off_t tell() const {
- off_t res = -1;
- if (-1 == (res = ::ftell(_fp))) {
- char errmsg[64];
- LOG(WARNING) << "fail to tell file. [err= " << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "']";
- }
- return res;
- }
-
- off_t length() const;
-
- off_t seek(off_t offset, int whence) {
- off_t res = -1;
- if (-1 == (res = ::fseek(_fp, offset, whence))) {
- char errmsg[64];
- LOG(WARNING) << "fail to seek file. [err=" << strerror_r(errno, errmsg, 64)
- << " file_name='" << _file_name << "' offset=" << offset
- << " whence=" << whence << "]";
- }
- return res;
- }
-
- const std::string& file_name() { return _file_name; }
-
- int fd() { return ::fileno(_fp); }
-
-private:
- FILE* _fp;
- std::string _file_name;
-};
-
-typedef struct _FixedFileHeader {
- // the length of the entire file
- uint32_t file_length;
- // Checksum of the file's contents except the FileHeader
- uint32_t checksum;
- // Protobuf length of section
- uint32_t protobuf_length;
- // Checksum of Protobuf part
- uint32_t protobuf_checksum;
-} __attribute__((packed)) FixedFileHeader;
-
-typedef struct _FixedFileHeaderV2 {
- uint64_t magic_number;
- uint32_t version;
- // the length of the entire file
- uint64_t file_length;
- // Checksum of the file's contents except the FileHeader
- uint32_t checksum;
- // Protobuf length of section
- uint64_t protobuf_length;
- // Checksum of Protobuf part
- uint32_t protobuf_checksum;
-} __attribute__((packed)) FixedFileHeaderV2;
-
-template <typename MessageType, typename ExtraType = uint32_t,
- typename FileHandlerType = FileHandler>
-class FileHeader {
-public:
- FileHeader() {
- memset(&_fixed_file_header, 0, sizeof(_fixed_file_header));
- memset(&_extra_fixed_header, 0, sizeof(_extra_fixed_header));
- _fixed_file_header_size = sizeof(_fixed_file_header);
- }
- ~FileHeader() {}
-
- // To calculate the length of the proto part, it needs to be called after the proto is operated, and prepare must be called before calling serialize
- Status prepare(FileHandlerType* file_handler);
-
- // call prepare() first, serialize() will write fixed header and protobuffer.
- // Write the header to the starting position of the incoming file handle
- Status serialize(FileHandlerType* file_handler);
-
- // read from file, validate file length, signature and alder32 of protobuffer.
- // Read the header from the beginning of the incoming file handle
- Status unserialize(FileHandlerType* file_handler);
-
- // Check the validity of Header
- // it is actually call unserialize().
- Status validate(const std::string& filename);
-
- uint64_t file_length() const { return _fixed_file_header.file_length; }
- uint32_t checksum() const { return _fixed_file_header.checksum; }
- const ExtraType& extra() const { return _extra_fixed_header; }
- ExtraType* mutable_extra() { return &_extra_fixed_header; }
- const MessageType& message() const { return _proto; }
- MessageType* mutable_message() { return &_proto; }
- uint64_t size() const {
- return _fixed_file_header_size + sizeof(_extra_fixed_header) +
- _fixed_file_header.protobuf_length;
- }
-
- void set_file_length(uint64_t file_length) { _fixed_file_header.file_length = file_length; }
- void set_checksum(uint32_t checksum) { _fixed_file_header.checksum = checksum; }
-
-private:
- FixedFileHeaderV2 _fixed_file_header;
- uint32_t _fixed_file_header_size;
-
- std::string _proto_string;
- ExtraType _extra_fixed_header;
- MessageType _proto;
-};
-
-// FileHandler implementation
-template <typename MessageType, typename ExtraType, typename FileHandlerType>
-Status FileHeader<MessageType, ExtraType, FileHandlerType>::prepare(FileHandlerType* file_handler) {
- if (nullptr == file_handler) {
- return Status::Error<ErrorCode::INVALID_ARGUMENT>();
- }
-
- // Use the file name as Signature to prevent problems caused by some misoperations
- // _proto.set_signature(basename(file_handler->file_name().c_str()));
-
- try {
- if (!_proto.SerializeToString(&_proto_string)) {
- LOG(WARNING) << "serialize file header to string error. [path='"
- << file_handler->file_name() << "']";
- return Status::Error<ErrorCode::SERIALIZE_PROTOBUF_ERROR>();
- }
- } catch (...) {
- LOG(WARNING) << "serialize file header to string error. [path='"
- << file_handler->file_name() << "']";
- return Status::Error<ErrorCode::SERIALIZE_PROTOBUF_ERROR>();
- }
-
- _fixed_file_header.protobuf_checksum =
- olap_adler32(olap_adler32_init(), _proto_string.c_str(), _proto_string.size());
-
- _fixed_file_header.checksum = 0;
- _fixed_file_header.protobuf_length = _proto_string.size();
- _fixed_file_header.file_length = size();
- _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED;
- _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER;
-
- return Status::OK();
-}
-
-template <typename MessageType, typename ExtraType, typename FileHandlerType>
-Status FileHeader<MessageType, ExtraType, FileHandlerType>::serialize(
- FileHandlerType* file_handler) {
- if (nullptr == file_handler) {
- return Status::Error<ErrorCode::INVALID_ARGUMENT>();
- }
-
- // write to file
- if (!file_handler->pwrite(&_fixed_file_header, _fixed_file_header_size, 0)) {
- char errmsg[64];
- LOG(WARNING) << "fail to write fixed header to file. [file='" << file_handler->file_name()
- << "' err=" << strerror_r(errno, errmsg, 64) << "]";
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- if (!file_handler->pwrite(&_extra_fixed_header, sizeof(_extra_fixed_header),
- _fixed_file_header_size)) {
- char errmsg[64];
- LOG(WARNING) << "fail to write extra fixed header to file. [file='"
- << file_handler->file_name() << "' err=" << strerror_r(errno, errmsg, 64)
- << "]";
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- if (!file_handler->pwrite(_proto_string.c_str(), _proto_string.size(),
- _fixed_file_header_size + sizeof(_extra_fixed_header))) {
- char errmsg[64];
- LOG(WARNING) << "fail to write proto header to file. [file='" << file_handler->file_name()
- << "' err='" << strerror_r(errno, errmsg, 64) << "']";
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- return Status::OK();
-}
-
-template <typename MessageType, typename ExtraType, typename FileHandlerType>
-Status FileHeader<MessageType, ExtraType, FileHandlerType>::unserialize(
- FileHandlerType* file_handler) {
- if (nullptr == file_handler) {
- return Status::Error<ErrorCode::INVALID_ARGUMENT>();
- }
-
- off_t real_file_length = 0;
- uint32_t real_protobuf_checksum = 0;
-
- if (!file_handler->pread(&_fixed_file_header, _fixed_file_header_size, 0)) {
- char errmsg[64];
- LOG(WARNING) << "fail to load header structure from file. file="
- << file_handler->file_name() << ", error=" << strerror_r(errno, errmsg, 64);
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- if (_fixed_file_header.magic_number != OLAP_FIX_HEADER_MAGIC_NUMBER) {
- VLOG_TRACE << "old fix header found, magic num=" << _fixed_file_header.magic_number;
- FixedFileHeader tmp_header;
-
- if (!file_handler->pread(&tmp_header, sizeof(tmp_header), 0)) {
- char errmsg[64];
- LOG(WARNING) << "fail to load header structure from file. file="
- << file_handler->file_name()
- << ", error=" << strerror_r(errno, errmsg, 64);
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- _fixed_file_header.file_length = tmp_header.file_length;
- _fixed_file_header.checksum = tmp_header.checksum;
- _fixed_file_header.protobuf_length = tmp_header.protobuf_length;
- _fixed_file_header.protobuf_checksum = tmp_header.protobuf_checksum;
- _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER;
- _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED;
-
- _fixed_file_header_size = sizeof(tmp_header);
- }
-
- VLOG_NOTICE << "fix head loaded. file_length=" << _fixed_file_header.file_length
- << ", checksum=" << _fixed_file_header.checksum
- << ", protobuf_length=" << _fixed_file_header.protobuf_length
- << ", magic_number=" << _fixed_file_header.magic_number
- << ", version=" << _fixed_file_header.version;
-
- if (!file_handler->pread(&_extra_fixed_header, sizeof(_extra_fixed_header),
- _fixed_file_header_size)) {
- char errmsg[64];
- LOG(WARNING) << "fail to load extra fixed header from file. file="
- << file_handler->file_name() << ", error=" << strerror_r(errno, errmsg, 64);
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- std::unique_ptr<char[]> buf(new (std::nothrow) char[_fixed_file_header.protobuf_length]);
-
- if (nullptr == buf.get()) {
- char errmsg[64];
- LOG(WARNING) << "malloc protobuf buf error. file=" << file_handler->file_name()
- << ", error=" << strerror_r(errno, errmsg, 64);
- return Status::Error<ErrorCode::MEM_ALLOC_FAILED>();
- }
-
- if (!file_handler->pread(buf.get(), _fixed_file_header.protobuf_length,
- _fixed_file_header_size + sizeof(_extra_fixed_header))) {
- char errmsg[64];
- LOG(WARNING) << "fail to load protobuf from file. file=" << file_handler->file_name()
- << ", error=" << strerror_r(errno, errmsg, 64);
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- real_file_length = file_handler->length();
-
- if (file_length() != static_cast<uint64_t>(real_file_length)) {
- LOG(WARNING) << "file length is not match. file=" << file_handler->file_name()
- << ", file_length=" << file_length()
- << ", real_file_length=" << real_file_length;
- return Status::Error<ErrorCode::FILE_DATA_ERROR>();
- }
-
- // check proto checksum
- real_protobuf_checksum =
- olap_adler32(olap_adler32_init(), buf.get(), _fixed_file_header.protobuf_length);
-
- if (real_protobuf_checksum != _fixed_file_header.protobuf_checksum) {
- LOG(WARNING) << "checksum is not match. file=" << file_handler->file_name()
- << ", expect=" << _fixed_file_header.protobuf_checksum
- << ", actual=" << real_protobuf_checksum;
- return Status::Error<ErrorCode::CHECKSUM_ERROR>();
- }
-
- try {
- std::string protobuf_str(buf.get(), _fixed_file_header.protobuf_length);
-
- if (!_proto.ParseFromString(protobuf_str)) {
- LOG(WARNING) << "fail to parse file content to protobuf object. file="
- << file_handler->file_name();
- return Status::Error<ErrorCode::PARSE_PROTOBUF_ERROR>();
- }
- } catch (...) {
- LOG(WARNING) << "fail to load protobuf. file='" << file_handler->file_name();
- return Status::Error<ErrorCode::PARSE_PROTOBUF_ERROR>();
- }
-
- return Status::OK();
-}
-
-template <typename MessageType, typename ExtraType, typename FileHandlerType>
-Status FileHeader<MessageType, ExtraType, FileHandlerType>::validate(const std::string& filename) {
- FileHandler file_handler;
- Status res = Status::OK();
-
- if (!file_handler.open(filename.c_str(), O_RDONLY)) {
- char errmsg[64];
- LOG(WARNING) << "fail to open file. [file='" << filename
- << "' err=" << strerror_r(errno, errmsg, 64) << "]";
- return Status::Error<ErrorCode::IO_ERROR>();
- }
-
- if (!(res = unserialize(&file_handler))) {
- LOG(WARNING) << "unserialize file header error. [file='" << filename << "']";
- return res;
- }
-
- return Status::OK();
-}
-
-} // namespace doris
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 62422f1410..adb1a5feb0 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -20,6 +20,7 @@
#include <memory>
#include "common/status.h"
+#include "io/io_common.h"
#include "olap/block_column_predicate.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
@@ -34,25 +35,6 @@ class RowCursor;
class Schema;
class ColumnPredicate;
-struct IOContext {
- IOContext() = default;
-
- IOContext(const TUniqueId* query_id_, FileCacheStatistics* stats_, bool is_presistent_,
- bool use_disposable_cache_, bool read_segment_index_, bool enable_file_cache)
- : query_id(query_id_),
- is_persistent(is_presistent_),
- use_disposable_cache(use_disposable_cache_),
- read_segment_index(read_segment_index_),
- file_cache_stats(stats_),
- enable_file_cache(enable_file_cache) {}
- ReaderType reader_type;
- const TUniqueId* query_id = nullptr;
- bool is_persistent = false;
- bool use_disposable_cache = false;
- bool read_segment_index = false;
- FileCacheStatistics* file_cache_stats = nullptr;
- bool enable_file_cache = true;
-};
namespace vectorized {
struct IteratorRowRef;
};
@@ -117,7 +99,7 @@ public:
bool read_orderby_key_reverse = false;
// columns for orderby keys
std::vector<uint32_t>* read_orderby_key_columns = nullptr;
- IOContext io_ctx;
+ io::IOContext io_ctx;
vectorized::VExpr* remaining_vconjunct_root = nullptr;
vectorized::VExprContext* common_vexpr_ctxs_pushdown = nullptr;
const std::set<int32_t>* output_columns = nullptr;
diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h
index a3d9b54e1c..467c126304 100644
--- a/be/src/olap/merger.h
+++ b/be/src/olap/merger.h
@@ -17,6 +17,7 @@
#pragma once
+#include "io/io_common.h"
#include "olap/olap_define.h"
#include "olap/rowid_conversion.h"
#include "olap/rowset/rowset_writer.h"
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 5dfbb96a8b..e45761fe58 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -33,6 +33,7 @@
#include "env/env.h"
#include "gen_cpp/Types_types.h"
+#include "io/io_common.h"
#include "olap/olap_define.h"
#include "util/hash_util.hpp"
#include "util/uid_util.h"
@@ -169,16 +170,6 @@ enum PushType {
PUSH_NORMAL_V2 = 4, // for spark load
};
-enum ReaderType {
- READER_QUERY = 0,
- READER_ALTER_TABLE = 1,
- READER_BASE_COMPACTION = 2,
- READER_CUMULATIVE_COMPACTION = 3,
- READER_CHECKSUM = 4,
- READER_COLD_DATA_COMPACTION = 5,
- READER_SEGMENT_COMPACTION = 6,
-};
-
constexpr bool field_is_slice_type(const FieldType& field_type) {
return field_type == OLAP_FIELD_TYPE_VARCHAR || field_type == OLAP_FIELD_TYPE_CHAR ||
field_type == OLAP_FIELD_TYPE_STRING;
@@ -264,17 +255,6 @@ using KeyRange = std::pair<WrapperField*, WrapperField*>;
static const int GENERAL_DEBUG_COUNT = 0;
-struct FileCacheStatistics {
- int64_t num_io_total = 0;
- int64_t num_io_hit_cache = 0;
- int64_t num_io_bytes_read_total = 0;
- int64_t num_io_bytes_read_from_file_cache = 0;
- int64_t num_io_bytes_read_from_write_cache = 0;
- int64_t num_io_written_in_file_cache = 0;
- int64_t num_io_bytes_written_in_file_cache = 0;
- int64_t num_io_bytes_skip_cache = 0;
-};
-
// ReaderStatistics used to collect statistics when scan data from storage
struct OlapReaderStatistics {
int64_t io_ns = 0;
@@ -373,7 +353,7 @@ struct OlapReaderStatistics {
// SCOPED_RAW_TIMER(&_stats->general_debug_ns[1]);
int64_t general_debug_ns[GENERAL_DEBUG_COUNT] = {};
- FileCacheStatistics file_cache_stats;
+ io::FileCacheStatistics file_cache_stats;
int64_t load_segments_timer = 0;
};
diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h
index b852e68bb2..8546fbf749 100644
--- a/be/src/olap/push_handler.h
+++ b/be/src/olap/push_handler.h
@@ -25,7 +25,6 @@
#include "gen_cpp/AgentService_types.h"
#include "gen_cpp/MasterService_types.h"
#include "gen_cpp/PaloInternalService_types.h"
-#include "olap/file_helper.h"
#include "olap/merger.h"
#include "olap/olap_common.h"
#include "olap/row_cursor.h"
diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h
index d286bf42a6..cfee9ee10a 100644
--- a/be/src/olap/reader.h
+++ b/be/src/olap/reader.h
@@ -22,6 +22,7 @@
#include "exprs/bitmapfilter_predicate.h"
#include "exprs/function_filter.h"
#include "exprs/hybrid_set.h"
+#include "io/io_common.h"
#include "olap/delete_handler.h"
#include "olap/row_cursor.h"
#include "olap/rowset/rowset_reader.h"
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index 4f459d8365..a8cfb8e022 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -217,6 +217,10 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id,
if (!fs) {
return Status::Error<INIT_FAILED>();
}
+ if (fs->type() != io::FileSystemType::LOCAL) {
+ return Status::InternalError("should be local file system");
+ }
+ io::LocalFileSystem* local_fs = (io::LocalFileSystem*)fs.get();
for (int i = 0; i < num_segments(); ++i) {
auto dst_path = segment_file_path(dir, new_rowset_id, i + new_rowset_start_seg_id);
// TODO(lingbin): use Env API? or EnvUtil?
@@ -228,7 +232,7 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id,
auto src_path = segment_file_path(i);
// TODO(lingbin): how external storage support link?
// use copy? or keep refcount to avoid being delete?
- if (!fs->link_file(src_path, dst_path).ok()) {
+ if (!local_fs->link_file(src_path, dst_path).ok()) {
LOG(WARNING) << "fail to create hard link. from=" << src_path << ", "
<< "to=" << dst_path << ", errno=" << Errno::no();
return Status::Error<OS_ERROR>();
@@ -244,7 +248,7 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id,
InvertedIndexDescriptor::get_index_file_name(dst_path,
index_meta->index_id());
- if (!fs->link_file(inverted_index_src_file_path, inverted_index_dst_file_path)
+ if (!local_fs->link_file(inverted_index_src_file_path, inverted_index_dst_file_path)
.ok()) {
LOG(WARNING) << "fail to create hard link. from="
<< inverted_index_src_file_path << ", "
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index 7671a75d43..825c6dbcd1 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -18,6 +18,7 @@
#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
+#include "io/io_common.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
#include "runtime/runtime_state.h"
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index ac064de0a3..62601a1e6e 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -26,6 +26,7 @@
#include "common/status.h" // for Status
#include "gen_cpp/segment_v2.pb.h" // for ColumnMetaPB
#include "io/fs/file_reader.h"
+#include "io/io_common.h"
#include "olap/block_column_predicate.h"
#include "olap/column_predicate.h"
#include "olap/iterators.h"
@@ -76,7 +77,7 @@ struct ColumnIteratorOptions {
// page types are divided into DATA_PAGE & INDEX_PAGE
// INDEX_PAGE including index_page, dict_page and short_key_page
PageTypePB type;
- IOContext io_ctx;
+ io::IOContext io_ctx;
void sanity_check() const {
CHECK_NOTNULL(file_reader);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
index 4f966bf44d..26b509ff7f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
@@ -19,6 +19,7 @@
#include "CLucene/SharedHeader.h"
#include "CLucene/StdHeader.h"
+#include "io/fs/file_writer.h"
#include "olap/iterators.h"
#include "util/md5.h"
@@ -237,7 +238,7 @@ bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, co
}
SharedHandle* h = _CLNEW SharedHandle(path);
- if (!fs->open_file(path, &h->_reader, nullptr).ok()) {
+ if (!fs->open_file(path, &h->_reader).ok()) {
error.set(CL_ERR_IO, "open file error");
}
@@ -335,8 +336,7 @@ void DorisCompoundDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_
Slice result {b, (size_t)len};
size_t bytes_read = 0;
- IOContext io_ctx;
- if (!_handle->_reader->read_at(_pos, result, io_ctx, &bytes_read).ok()) {
+ if (!_handle->_reader->read_at(_pos, result, &bytes_read).ok()) {
_CLTHROWA(CL_ERR_IO, "read past EOF");
}
bufferLength = len;
@@ -512,10 +512,11 @@ bool DorisCompoundDirectory::list(std::vector<std::string>* names) const {
CND_PRECONDITION(!directory.empty(), "directory is not open");
char fl[CL_MAX_DIR];
priv_getFN(fl, "");
- std::vector<std::filesystem::path> paths;
- RETURN_IF_ERROR(fs->list(fl, &paths));
- for (auto path : paths) {
- names->push_back(path.string());
+ std::vector<io::FileInfo> files;
+ bool exists;
+ RETURN_IF_ERROR(fs->list(fl, true, &files, &exists));
+ for (auto& file : files) {
+ names->push_back(file.file_name);
}
return true;
}
diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp
index 697775dcdd..7eb3dd0549 100644
--- a/be/src/olap/rowset/segment_v2/page_io.cpp
+++ b/be/src/olap/rowset/segment_v2/page_io.cpp
@@ -137,8 +137,8 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle*
{
SCOPED_RAW_TIMER(&opts.stats->io_ns);
size_t bytes_read = 0;
- RETURN_IF_ERROR(opts.file_reader->read_at(opts.page_pointer.offset, page_slice, opts.io_ctx,
- &bytes_read));
+ RETURN_IF_ERROR(opts.file_reader->read_at(opts.page_pointer.offset, page_slice, &bytes_read,
+ &opts.io_ctx));
DCHECK_EQ(bytes_read, page_size);
opts.stats->compressed_bytes_read += page_size;
}
diff --git a/be/src/olap/rowset/segment_v2/page_io.h b/be/src/olap/rowset/segment_v2/page_io.h
index 14f93021bf..06e345c27a 100644
--- a/be/src/olap/rowset/segment_v2/page_io.h
+++ b/be/src/olap/rowset/segment_v2/page_io.h
@@ -71,7 +71,7 @@ struct PageReadOptions {
// index_page should not be pre-decoded
bool pre_decode = true;
- IOContext io_ctx;
+ io::IOContext io_ctx;
void sanity_check() const {
CHECK_NOTNULL(file_reader);
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index ea0e6b44cd..f59b716361 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -50,14 +50,14 @@ Status Segment::open(io::FileSystemSPtr fs, const std::string& path, uint32_t se
std::shared_ptr<Segment>* output) {
io::FileReaderSPtr file_reader;
#ifndef BE_TEST
- RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader, nullptr));
+ RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader));
#else
// be ut use local file reader instead of remote file reader while use remote cache
if (!config::file_cache_type.empty()) {
- RETURN_IF_ERROR(io::global_local_filesystem()->open_file(path, reader_options, &file_reader,
- nullptr));
+ RETURN_IF_ERROR(
+ io::global_local_filesystem()->open_file(path, reader_options, &file_reader));
} else {
- RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader, nullptr));
+ RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader));
}
#endif
@@ -148,9 +148,7 @@ Status Segment::_parse_footer() {
uint8_t fixed_buf[12];
size_t bytes_read = 0;
- IOContext io_ctx;
- RETURN_IF_ERROR(
- _file_reader->read_at(file_size - 12, Slice(fixed_buf, 12), io_ctx, &bytes_read));
+ RETURN_IF_ERROR(_file_reader->read_at(file_size - 12, Slice(fixed_buf, 12), &bytes_read));
DCHECK_EQ(bytes_read, 12);
// validate magic number
@@ -170,8 +168,7 @@ Status Segment::_parse_footer() {
std::string footer_buf;
footer_buf.resize(footer_length);
- RETURN_IF_ERROR(
- _file_reader->read_at(file_size - 12 - footer_length, footer_buf, io_ctx, &bytes_read));
+ RETURN_IF_ERROR(_file_reader->read_at(file_size - 12 - footer_length, footer_buf, &bytes_read));
DCHECK_EQ(bytes_read, footer_length);
// validate footer PB's checksum
diff --git a/be/src/olap/storage_policy.h b/be/src/olap/storage_policy.h
index 02d86603fa..04f05540ae 100644
--- a/be/src/olap/storage_policy.h
+++ b/be/src/olap/storage_policy.h
@@ -52,7 +52,7 @@ void delete_storage_policy(int64_t id);
std::vector<std::pair<int64_t, int64_t>> get_storage_policy_ids();
struct StorageResource {
- io::FileSystemSPtr fs;
+ io::RemoteFileSystemSPtr fs;
int64_t version = -1;
};
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index f32e0b077b..d30b5b0ba4 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1819,13 +1819,12 @@ Status Tablet::_read_cooldown_meta(const std::shared_ptr<io::RemoteFileSystem>&
TabletMetaPB* tablet_meta_pb) {
std::string remote_meta_path =
remote_tablet_meta_path(tablet_id(), _cooldown_replica_id, _cooldown_term);
- IOContext io_ctx;
io::FileReaderSPtr tablet_meta_reader;
- RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader, &io_ctx));
+ RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader));
auto file_size = tablet_meta_reader->size();
size_t bytes_read;
auto buf = std::unique_ptr<uint8_t[]>(new uint8_t[file_size]);
- RETURN_IF_ERROR(tablet_meta_reader->read_at(0, {buf.get(), file_size}, io_ctx, &bytes_read));
+ RETURN_IF_ERROR(tablet_meta_reader->read_at(0, {buf.get(), file_size}, &bytes_read));
tablet_meta_reader->close();
if (!tablet_meta_pb->ParseFromArray(buf.get(), file_size)) {
return Status::InternalError("malformed tablet meta");
@@ -2131,7 +2130,7 @@ void Tablet::remove_unused_remote_files() {
req.__isset.confirm_list = true;
// tablet_id -> [fs, unused_remote_files]
using unused_remote_files_buffer_t = std::unordered_map<
- int64_t, std::pair<std::shared_ptr<io::RemoteFileSystem>, std::vector<io::Path>>>;
+ int64_t, std::pair<std::shared_ptr<io::RemoteFileSystem>, std::vector<io::FileInfo>>>;
unused_remote_files_buffer_t buffer;
int64_t num_files_in_buffer = 0;
// assume a filename is 0.1KB, buffer size should not larger than 100MB
@@ -2161,15 +2160,17 @@ void Tablet::remove_unused_remote_files() {
return;
}
- std::vector<io::Path> files;
+ std::vector<io::FileInfo> files;
// FIXME(plat1ko): What if user reset resource in storage policy to another resource?
// Maybe we should also list files in previously uploaded resources.
- st = dest_fs->list(remote_tablet_path(t->tablet_id()), &files);
+ bool exists = true;
+ st = dest_fs->list(io::Path(remote_tablet_path(t->tablet_id())), true, &files, &exists);
if (!st.ok()) {
LOG(WARNING) << "encounter error when remove unused remote files, tablet_id="
<< t->tablet_id() << " : " << st;
+ return;
}
- if (files.empty()) {
+ if (!exists || files.empty()) {
return;
}
// get all cooldowned rowsets
@@ -2197,8 +2198,8 @@ void Tablet::remove_unused_remote_files() {
fmt::format("{}.{}.meta", cooldown_replica_id, cooldown_term);
// filter out the paths that should be reserved
// clang-format off
- files.erase(std::remove_if(files.begin(), files.end(), [&](io::Path& path) {
- const std::string& path_str = path.native();
+ files.erase(std::remove_if(files.begin(), files.end(), [&](io::FileInfo& info) {
+ const std::string& path_str = info.file_name;
if (StringPiece(path_str).ends_with(".meta")) {
return path_str == remote_meta_path;
}
@@ -2251,10 +2252,14 @@ void Tablet::remove_unused_remote_files() {
<< " tablet_id=" << id;
io::Path dir("data/" + std::to_string(id));
for (auto& file : files) {
- file = dir / file;
- LOG(INFO) << "delete unused file: " << file.native();
+ auto delete_path = dir / io::Path(file.file_name);
+ LOG(INFO) << "delete unused file: " << delete_path.native();
+ }
+ std::vector<io::Path> file_names;
+ for (auto& info : files) {
+ file_names.emplace_back(info.file_name);
}
- st = fs->batch_delete(files);
+ st = fs->batch_delete(file_names);
if (!st.ok()) {
LOG(WARNING) << "failed to delete unused files, tablet_id=" << id << " : "
<< st;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 5bc7019fda..e501e17f39 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -730,7 +730,7 @@ Status TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tablet_
if (!status.ok()) {
LOG(WARNING) << "fail to load tablet because can not parse meta_binary string. "
<< "tablet_id=" << tablet_id << ", schema_hash=" << schema_hash
- << ", path=" << data_dir->path();
+ << ", path=" << data_dir->path() << ", status: " << status;
return Status::Error<HEADER_PB_PARSE_FAILED>();
}
tablet_meta->init_rs_metas_fs(data_dir->fs());
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 4cea4baa4f..7a6ec8c765 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -20,7 +20,8 @@
#include <sstream>
#include "common/consts.h"
-#include "olap/file_helper.h"
+#include "io/fs/file_writer.h"
+#include "olap/file_header.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/tablet_meta_manager.h"
@@ -321,22 +322,9 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco
}
Status TabletMeta::create_from_file(const string& file_path) {
- FileHeader<TabletMetaPB> file_header;
- FileHandler file_handler;
-
- auto open_status = file_handler.open(file_path, O_RDONLY);
-
- if (!open_status.ok()) {
- LOG(WARNING) << "fail to open ordinal file. file=" << file_path;
- return open_status;
- }
-
- // In file_header.unserialize(), it validates file length, signature, checksum of protobuf.
- if (file_header.unserialize(&file_handler) != Status::OK()) {
- LOG(WARNING) << "fail to unserialize tablet_meta. file='" << file_path;
- return Status::Error<PARSE_PROTOBUF_ERROR>();
- }
-
+ FileHeader<TabletMetaPB> file_header(file_path);
+ // In file_header.deserialize(), it validates file length, signature, checksum of protobuf.
+ RETURN_IF_ERROR(file_header.deserialize());
TabletMetaPB tablet_meta_pb;
try {
tablet_meta_pb.CopyFrom(file_header.message());
@@ -398,30 +386,15 @@ Status TabletMeta::save(const string& file_path) {
Status TabletMeta::save(const string& file_path, const TabletMetaPB& tablet_meta_pb) {
DCHECK(!file_path.empty());
-
- FileHeader<TabletMetaPB> file_header;
- FileHandler file_handler;
-
- auto open_status =
- file_handler.open_with_mode(file_path, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR);
- if (!open_status.ok()) {
- LOG(WARNING) << "fail to open header file. file='" << file_path;
- return open_status;
- }
-
+ FileHeader<TabletMetaPB> file_header(file_path);
try {
file_header.mutable_message()->CopyFrom(tablet_meta_pb);
} catch (...) {
LOG(WARNING) << "fail to copy protocol buffer object. file='" << file_path;
return Status::Error<ErrorCode::INTERNAL_ERROR>();
}
-
- if (file_header.prepare(&file_handler) != Status::OK() ||
- file_header.serialize(&file_handler) != Status::OK()) {
- LOG(WARNING) << "fail to serialize to file header. file='" << file_path;
- return Status::Error<SERIALIZE_PROTOBUF_ERROR>();
- }
-
+ RETURN_IF_ERROR(file_header.prepare());
+ RETURN_IF_ERROR(file_header.serialize());
return Status::OK();
}
diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp
index 0ea8c6450d..755f73c08e 100644
--- a/be/src/olap/utils.cpp
+++ b/be/src/olap/utils.cpp
@@ -33,7 +33,6 @@
#include <string>
#include <vector>
-#include "olap/file_helper.h"
#include "util/file_utils.h"
#ifdef DORIS_WITH_LZO
@@ -47,6 +46,9 @@
#include "common/status.h"
#include "env/env.h"
#include "gutil/strings/substitute.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/local_file_system.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "util/errno.h"
@@ -449,13 +451,7 @@ Status read_write_test_file(const string& test_file_path) {
return Status::Error<IO_ERROR>();
}
}
- Status res = Status::OK();
- FileHandler file_handler;
- if ((res = file_handler.open_with_mode(test_file_path.c_str(), O_RDWR | O_CREAT | O_SYNC,
- S_IRUSR | S_IWUSR)) != Status::OK()) {
- LOG(WARNING) << "fail to create test file. path=" << test_file_path;
- return res;
- }
+
const size_t TEST_FILE_BUF_SIZE = 4096;
const size_t DIRECT_IO_ALIGNMENT = 512;
char* write_test_buff = nullptr;
@@ -476,30 +472,24 @@ Status read_write_test_file(const string& test_file_path) {
int32_t tmp_value = rand_r(&rand_seed);
write_test_buff[i] = static_cast<char>(tmp_value);
}
- if (!(res = file_handler.pwrite(write_buff.get(), TEST_FILE_BUF_SIZE, SEEK_SET))) {
- LOG(WARNING) << "fail to write test file. [file_name=" << test_file_path << "]";
- return res;
- }
- if ((res = file_handler.pread(read_buff.get(), TEST_FILE_BUF_SIZE, SEEK_SET)) != Status::OK()) {
- LOG(WARNING) << "fail to read test file. [file_name=" << test_file_path << "]";
- return res;
- }
+
+ // write file
+ io::FileWriterPtr file_writer;
+ RETURN_IF_ERROR(io::global_local_filesystem()->create_file(test_file_path, &file_writer));
+ RETURN_IF_ERROR(file_writer->append({write_buff.get(), TEST_FILE_BUF_SIZE}));
+ RETURN_IF_ERROR(file_writer->close());
+ // read file
+ io::FileReaderSPtr file_reader;
+ RETURN_IF_ERROR(io::global_local_filesystem()->open_file(test_file_path, &file_reader));
+ size_t bytes_read = 0;
+ RETURN_IF_ERROR(file_reader->read_at(0, {read_buff.get(), TEST_FILE_BUF_SIZE}, &bytes_read));
if (memcmp(write_buff.get(), read_buff.get(), TEST_FILE_BUF_SIZE) != 0) {
LOG(WARNING) << "the test file write_buf and read_buf not equal, [file_name = "
<< test_file_path << "]";
return Status::Error<TEST_FILE_ERROR>();
}
- if ((res = file_handler.close()) != Status::OK()) {
- LOG(WARNING) << "fail to close test file. [file_name=" << test_file_path << "]";
- return res;
- }
- if (remove(test_file_path.c_str()) != 0) {
- char errmsg[64];
- VLOG_NOTICE << "fail to delete test file. [err='" << strerror_r(errno, errmsg, 64)
- << "' path='" << test_file_path << "']";
- return Status::Error<IO_ERROR>();
- }
- return res;
+ // delete file
+ return io::global_local_filesystem()->delete_file(test_file_path);
}
bool check_datapath_rw(const string& path) {
diff --git a/be/src/runtime/routine_load/routine_load_task_executor.cpp b/be/src/runtime/routine_load/routine_load_task_executor.cpp
index ae0422e675..3aa361d157 100644
--- a/be/src/runtime/routine_load/routine_load_task_executor.cpp
+++ b/be/src/runtime/routine_load/routine_load_task_executor.cpp
@@ -371,8 +371,7 @@ Status RoutineLoadTaskExecutor::_execute_plan_for_test(std::shared_ptr<StreamLoa
int64_t len = 1;
size_t read_bytes = 0;
Slice result((uint8_t*)&one, len);
- IOContext io_ctx;
- Status st = pipe->read_at(0, result, io_ctx, &read_bytes);
+ Status st = pipe->read_at(0, result, &read_bytes);
if (!st.ok()) {
LOG(WARNING) << "read failed";
ctx->promise.set_value(st);
diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp
index c4b2a390df..846ce688e2 100644
--- a/be/src/runtime/snapshot_loader.cpp
+++ b/be/src/runtime/snapshot_loader.cpp
@@ -26,16 +26,17 @@
#include "gen_cpp/HeartbeatService_types.h"
#include "gen_cpp/PaloBrokerService_types.h"
#include "gen_cpp/TPaloBrokerService.h"
-#include "olap/file_helper.h"
+#include "io/fs/broker_file_system.h"
+#include "io/fs/hdfs_file_system.h"
+#include "io/fs/s3_file_system.h"
+#include "io/hdfs_builder.h"
#include "olap/snapshot_manager.h"
#include "olap/storage_engine.h"
#include "olap/tablet.h"
#include "runtime/broker_mgr.h"
#include "runtime/exec_env.h"
-#include "util/broker_storage_backend.h"
#include "util/file_utils.h"
-#include "util/hdfs_storage_backend.h"
-#include "util/s3_storage_backend.h"
+#include "util/s3_uri.h"
#include "util/thrift_rpc_helper.h"
namespace doris {
@@ -46,29 +47,42 @@ SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id)
_task_id(task_id),
_broker_addr(TNetworkAddress()),
_prop(std::map<std::string, std::string>()),
- _storage_backend(nullptr) {}
+ _remote_fs(nullptr) {}
SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id,
const TNetworkAddress& broker_addr,
- const std::map<std::string, std::string>& prop,
- TStorageBackendType::type type)
- : _env(env), _job_id(job_id), _task_id(task_id), _broker_addr(broker_addr), _prop(prop) {
+ const std::map<std::string, std::string>& prop)
+ : _env(env), _job_id(job_id), _task_id(task_id), _broker_addr(broker_addr), _prop(prop) {}
+
+Status SnapshotLoader::init(TStorageBackendType::type type, const std::string& location) {
if (TStorageBackendType::type::S3 == type) {
- _storage_backend.reset(new S3StorageBackend(_prop));
+ S3Conf s3_conf;
+ S3URI s3_uri(location);
+ RETURN_IF_ERROR(s3_uri.parse());
+ RETURN_IF_ERROR(S3ClientFactory::convert_properties_to_s3_conf(_prop, s3_uri, &s3_conf));
+ std::shared_ptr<io::S3FileSystem> fs;
+ RETURN_IF_ERROR(io::S3FileSystem::create(std::move(s3_conf), "", &fs));
+ _remote_fs = std::move(fs);
} else if (TStorageBackendType::type::HDFS == type) {
- _storage_backend.reset(new HDFSStorageBackend(_prop));
+ THdfsParams hdfs_params = parse_properties(_prop);
+ std::shared_ptr<io::HdfsFileSystem> fs;
+ RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs));
+ _remote_fs = std::move(fs);
} else if (TStorageBackendType::type::BROKER == type) {
- _storage_backend.reset(new BrokerStorageBackend(_env, _broker_addr, _prop));
+ std::shared_ptr<io::BrokerFileSystem> fs;
+ RETURN_IF_ERROR(io::BrokerFileSystem::create(_broker_addr, _prop, 0, &fs));
+ _remote_fs = std::move(fs);
} else {
- _storage_backend = nullptr;
+ return Status::InternalError("Unknown storage tpye: {}", type);
}
+ return Status::OK();
}
SnapshotLoader::~SnapshotLoader() = default;
Status SnapshotLoader::upload(const std::map<std::string, std::string>& src_to_dest_path,
std::map<int64_t, std::vector<std::string>>* tablet_files) {
- if (!_storage_backend) {
+ if (!_remote_fs) {
return Status::InternalError("Storage backend not initialized.");
}
LOG(INFO) << "begin to upload snapshot files. num: " << src_to_dest_path.size()
@@ -99,7 +113,7 @@ Status SnapshotLoader::upload(const std::map<std::string, std::string>& src_to_d
// 2.1 get existing files from remote path
std::map<std::string, FileStat> remote_files;
- RETURN_IF_ERROR(_storage_backend->list(dest_path, true, false, &remote_files));
+ RETURN_IF_ERROR(_list_with_checksum(dest_path, &remote_files));
for (auto& tmp : remote_files) {
VLOG_CRITICAL << "get remote file: " << tmp.first << ", checksum: " << tmp.second.md5;
@@ -151,8 +165,8 @@ Status SnapshotLoader::upload(const std::map<std::string, std::string>& src_to_d
// upload
std::string full_remote_file = dest_path + "/" + local_file;
std::string full_local_file = src_path + "/" + local_file;
- RETURN_IF_ERROR(_storage_backend->upload_with_checksum(full_local_file,
- full_remote_file, md5sum));
+ RETURN_IF_ERROR(
+ _remote_fs->upload_with_checksum(full_local_file, full_remote_file, md5sum));
} // end for each tablet's local files
tablet_files->emplace(tablet_id, local_files_with_checksum);
@@ -172,7 +186,7 @@ Status SnapshotLoader::upload(const std::map<std::string, std::string>& src_to_d
*/
Status SnapshotLoader::download(const std::map<std::string, std::string>& src_to_dest_path,
std::vector<int64_t>* downloaded_tablet_ids) {
- if (!_storage_backend) {
+ if (!_remote_fs) {
return Status::InternalError("Storage backend not initialized.");
}
LOG(INFO) << "begin to download snapshot files. num: " << src_to_dest_path.size()
@@ -213,7 +227,7 @@ Status SnapshotLoader::download(const std::map<std::string, std::string>& src_to
// 2.2. get remote files
std::map<std::string, FileStat> remote_files;
- RETURN_IF_ERROR(_storage_backend->list(remote_path, true, false, &remote_files));
+ RETURN_IF_ERROR(_list_with_checksum(remote_path, &remote_files));
if (remote_files.empty()) {
std::stringstream ss;
ss << "get nothing from remote path: " << remote_path;
@@ -287,7 +301,7 @@ Status SnapshotLoader::download(const std::map<std::string, std::string>& src_to
// remove file which will be downloaded now.
// this file will be added to local_files if it be downloaded successfully.
local_files.erase(find);
- RETURN_IF_ERROR(_storage_backend->download(full_remote_file, full_local_file));
+ RETURN_IF_ERROR(_remote_fs->download(full_remote_file, full_local_file));
// 3. check md5 of the downloaded file
std::string downloaded_md5sum;
@@ -628,4 +642,25 @@ Status SnapshotLoader::_report_every(int report_threshold, int* counter, int32_t
return Status::OK();
}
+Status SnapshotLoader::_list_with_checksum(const std::string& dir,
+ std::map<std::string, FileStat>* md5_files) {
+ bool exists = true;
+ std::vector<io::FileInfo> files;
+ RETURN_IF_ERROR(_remote_fs->list(dir, true, &files, &exists));
+ for (auto& tmp_file : files) {
+ io::Path path(tmp_file.file_name);
+ std::string file_name = path.filename();
+ size_t pos = file_name.find_last_of(".");
+ if (pos == std::string::npos || pos == file_name.size() - 1) {
+ // Not found checksum separator, ignore this file
+ continue;
+ }
+ FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1),
+ tmp_file.file_size};
+ md5_files->emplace(std::string(file_name, 0, pos), stat);
+ }
+
+ return Status::OK();
+}
+
} // end namespace doris
diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h
index 30eaa9f9f3..2451c07518 100644
--- a/be/src/runtime/snapshot_loader.h
+++ b/be/src/runtime/snapshot_loader.h
@@ -25,14 +25,19 @@
#include "common/status.h"
#include "gen_cpp/Types_types.h"
+#include "io/fs/remote_file_system.h"
#include "olap/tablet.h"
#include "runtime/client_cache.h"
namespace doris {
+struct FileStat {
+ std::string name;
+ std::string md5;
+ size_t size;
+};
class ExecEnv;
class StorageBackend;
-struct FileStat;
/*
* Upload:
@@ -59,11 +64,12 @@ public:
SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id);
SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id,
const TNetworkAddress& broker_addr,
- const std::map<std::string, std::string>& broker_prop,
- TStorageBackendType::type type);
+ const std::map<std::string, std::string>& broker_prop);
~SnapshotLoader();
+ Status init(TStorageBackendType::type type, const std::string& location);
+
Status upload(const std::map<std::string, std::string>& src_to_dest_path,
std::map<int64_t, std::vector<std::string>>* tablet_files);
@@ -92,13 +98,15 @@ private:
Status _report_every(int report_threshold, int* counter, int finished_num, int total_num,
TTaskType::type type);
+ Status _list_with_checksum(const std::string& dir, std::map<std::string, FileStat>* md5_files);
+
private:
ExecEnv* _env;
int64_t _job_id;
int64_t _task_id;
const TNetworkAddress _broker_addr;
const std::map<std::string, std::string> _prop;
- std::unique_ptr<StorageBackend> _storage_backend;
+ std::shared_ptr<io::RemoteFileSystem> _remote_fs;
};
} // end namespace doris
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 257509a97c..894c07e6eb 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -463,8 +463,8 @@ void PInternalServiceImpl::fetch_table_schema(google::protobuf::RpcController* c
std::unique_ptr<vectorized::GenericReader> reader(nullptr);
std::unique_ptr<RuntimeProfile> profile(new RuntimeProfile("FetchTableSchema"));
- IOContext io_ctx;
- FileCacheStatistics file_cache_statis;
+ io::IOContext io_ctx;
+ io::FileCacheStatistics file_cache_statis;
io_ctx.file_cache_stats = &file_cache_statis;
switch (params.format_type) {
case TFileFormatType::FORMAT_CSV_PLAIN:
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index 23e14e3b4c..b6d5f32e1e 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -50,7 +50,6 @@ set(UTIL_FILES
progress_updater.cpp
runtime_profile.cpp
static_asserts.cpp
- # storage_backend_mgr.cpp
string_parser.cpp
thrift_util.cpp
thrift_client.cpp
@@ -63,7 +62,6 @@ set(UTIL_FILES
mysql_row_buffer.cpp
error_util.cc
filesystem_util.cc
- broker_storage_backend.cpp
time.cpp
os_info.cpp
os_util.cpp
@@ -94,9 +92,7 @@ set(UTIL_FILES
zlib.cpp
pprof_utils.cpp
s3_uri.cpp
- s3_storage_backend.cpp
s3_util.cpp
- hdfs_storage_backend.cpp
hdfs_util.cpp
time_lut.cpp
cityhash102/city.cc
diff --git a/be/src/util/async_io.h b/be/src/util/async_io.h
index 6d324e4f2a..fce50e79b7 100644
--- a/be/src/util/async_io.h
+++ b/be/src/util/async_io.h
@@ -81,10 +81,10 @@ public:
cv.notify_one();
};
- if (file_type == io::FileSystemType::S3) {
- AsyncIO::instance().remote_thread_pool()->offer(task);
- } else {
+ if (file_type == io::FileSystemType::LOCAL) {
AsyncIO::instance().io_thread_pool()->offer(task);
+ } else {
+ AsyncIO::instance().remote_thread_pool()->offer(task);
}
cv.wait(l);
}
@@ -102,4 +102,4 @@ private:
PriorityThreadPool* remote_thread_pool() { return _remote_thread_pool; }
};
-} // end namespace doris
\ No newline at end of file
+} // end namespace doris
diff --git a/be/src/util/broker_storage_backend.cpp b/be/src/util/broker_storage_backend.cpp
deleted file mode 100644
index af2d5a7192..0000000000
--- a/be/src/util/broker_storage_backend.cpp
+++ /dev/null
@@ -1,417 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/broker_storage_backend.h"
-
-#include "env/env.h"
-#include "gen_cpp/FrontendService.h"
-#include "gen_cpp/FrontendService_types.h"
-#include "gen_cpp/HeartbeatService_types.h"
-#include "gen_cpp/PaloBrokerService_types.h"
-#include "gen_cpp/TPaloBrokerService.h"
-#include "io/broker_writer.h"
-#include "io/file_factory.h"
-#include "io/fs/file_reader.h"
-#include "io/fs/file_reader_options.h"
-#include "olap/file_helper.h"
-#include "olap/iterators.h"
-#include "runtime/client_cache.h"
-#include "runtime/exec_env.h"
-
-namespace doris {
-
-#ifdef BE_TEST
-inline BrokerServiceClientCache* client_cache(ExecEnv* env) {
- static BrokerServiceClientCache s_client_cache;
- return &s_client_cache;
-}
-#else
-inline BrokerServiceClientCache* client_cache(ExecEnv* env) {
- return env->broker_client_cache();
-}
-#endif
-
-void BrokerStorageBackend::_init_file_description(const std::string& remote) {
- _file_description.path = remote;
- _file_description.start_offset = 0;
- _file_description.file_size = 0;
-}
-
-BrokerStorageBackend::BrokerStorageBackend(ExecEnv* env, const TNetworkAddress& broker_addr,
- const std::map<std::string, std::string>& broker_prop)
- : _env(env), _broker_addr(broker_addr), _broker_prop(broker_prop) {}
-
-Status BrokerStorageBackend::download(const std::string& remote, const std::string& local) {
- // 1. open remote file for read
- std::shared_ptr<io::FileSystem> file_system;
- io::FileReaderSPtr broker_reader = nullptr;
- auto cache_policy = io::FileCachePolicy::NO_CACHE;
- IOContext io_ctx;
- if (config::enable_file_cache && io_ctx.enable_file_cache) {
- cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE;
- }
- io::FileBlockCachePathPolicy file_block_cache;
- io::FileReaderOptions reader_options(cache_policy, file_block_cache);
- _init_file_description(remote);
- RETURN_IF_ERROR(FileFactory::create_broker_reader(_broker_addr, _broker_prop, _file_description,
- &file_system, &broker_reader, reader_options,
- &io_ctx));
-
- // 2. remove the existing local file if exist
- if (std::filesystem::remove(local)) {
- VLOG(2) << "remove the previously exist local file: " << local;
- }
-
- // 3. open local file for write
- FileHandler file_handler;
- Status ost =
- file_handler.open_with_mode(local, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR);
- if (!ost.ok()) {
- return Status::InternalError("failed to open file: {}", local);
- }
-
- // 4. read remote and write to local
- VLOG(2) << "read remote file: " << remote << " to local: " << local;
- constexpr size_t buf_sz = 1024 * 1024;
- std::unique_ptr<uint8_t[]> read_buf(new uint8_t[buf_sz]);
- size_t write_offset = 0;
- size_t cur_offset = 0;
- while (true) {
- size_t read_len = 0;
- Slice file_slice(read_buf.get(), buf_sz);
- RETURN_IF_ERROR(broker_reader->read_at(cur_offset, file_slice, io_ctx, &read_len));
- cur_offset += read_len;
- if (read_len == 0) {
- break;
- }
-
- ost = file_handler.pwrite(read_buf.get(), read_len, write_offset);
- if (!ost.ok()) {
- return Status::InternalError("failed to write file: {}", local);
- }
- write_offset += read_len;
- } // file_handler should be closed before calculating checksum
-
- return Status::OK();
-}
-
-Status BrokerStorageBackend::direct_download(const std::string& remote, std::string* content) {
- return Status::IOError("broker direct_download not support ");
-}
-
-Status BrokerStorageBackend::upload(const std::string& local, const std::string& remote) {
- // read file and write to broker
- FileHandler file_handler;
- Status ost = file_handler.open(local, O_RDONLY);
- if (!ost.ok()) {
- return Status::InternalError("failed to open file: {}", local);
- }
-
- size_t file_len = file_handler.length();
- if (file_len == -1) {
- return Status::InternalError("failed to get length of file: {}", local);
- }
-
- // NOTICE: broker writer must be closed before calling rename
- std::vector<TNetworkAddress> broker_addrs;
- broker_addrs.push_back(_broker_addr);
- std::unique_ptr<BrokerWriter> broker_writer(
- new BrokerWriter(_env, broker_addrs, _broker_prop, remote, 0 /* offset */));
- RETURN_IF_ERROR(broker_writer->open());
-
- constexpr size_t buf_sz = 1024 * 1024;
- char read_buf[buf_sz];
- size_t left_len = file_len;
- size_t read_offset = 0;
- while (left_len > 0) {
- size_t read_len = left_len > buf_sz ? buf_sz : left_len;
- ost = file_handler.pread(read_buf, read_len, read_offset);
- if (!ost.ok()) {
- return Status::InternalError("failed to read file: {}", local);
- }
- // write through broker
- size_t write_len = 0;
- RETURN_IF_ERROR(broker_writer->write(reinterpret_cast<const uint8_t*>(read_buf), read_len,
- &write_len));
- DCHECK_EQ(write_len, read_len);
-
- read_offset += read_len;
- left_len -= read_len;
- }
-
- // close manually, because we need to check its close status
- RETURN_IF_ERROR(broker_writer->close());
-
- LOG(INFO) << "finished to write file via broker. file: " << local << ", length: " << file_len;
- return Status::OK();
-}
-
-Status BrokerStorageBackend::rename(const std::string& orig_name, const std::string& new_name) {
- Status status = Status::OK();
- BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms,
- &status);
- if (!status.ok()) {
- std::stringstream ss;
- ss << "failed to get broker client. "
- << "broker addr: " << _broker_addr << ". msg: " << status;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- try {
- TBrokerOperationStatus op_status;
- TBrokerRenamePathRequest rename_req;
- rename_req.__set_version(TBrokerVersion::VERSION_ONE);
- rename_req.__set_srcPath(orig_name);
- rename_req.__set_destPath(new_name);
- rename_req.__set_properties(_broker_prop);
-
- try {
- client->renamePath(op_status, rename_req);
- } catch (apache::thrift::transport::TTransportException& e) {
- RETURN_IF_ERROR(client.reopen());
- client->renamePath(op_status, rename_req);
- }
-
- if (op_status.statusCode != TBrokerOperationStatusCode::OK) {
- std::stringstream ss;
- ss << "Fail to rename file: " << orig_name << " to: " << new_name
- << " msg:" << op_status.message;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- } catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "Fail to rename file: " << orig_name << " to: " << new_name << " msg:" << e.what();
- LOG(WARNING) << ss.str();
- return Status::RpcError(ss.str());
- }
-
- LOG(INFO) << "finished to rename file. orig: " << orig_name << ", new: " << new_name;
-
- return status;
-}
-
-Status BrokerStorageBackend::rename_dir(const std::string& orig_name, const std::string& new_name) {
- return rename(orig_name, new_name);
-}
-
-Status BrokerStorageBackend::list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) {
- Status status = Status::OK();
- BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms,
- &status);
- if (!status.ok()) {
- std::stringstream ss;
- ss << "failed to get broker client. "
- << "broker addr: " << _broker_addr << ". msg: " << status;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- try {
- // get existing files from remote path
- TBrokerListResponse list_rep;
- TBrokerListPathRequest list_req;
- list_req.__set_version(TBrokerVersion::VERSION_ONE);
- list_req.__set_path(remote_path + "/*");
- list_req.__set_isRecursive(false);
- list_req.__set_properties(_broker_prop);
- list_req.__set_fileNameOnly(true); // we only need file name, not abs path
-
- try {
- client->listPath(list_rep, list_req);
- } catch (apache::thrift::transport::TTransportException& e) {
- RETURN_IF_ERROR(client.reopen());
- client->listPath(list_rep, list_req);
- }
-
- if (list_rep.opStatus.statusCode == TBrokerOperationStatusCode::FILE_NOT_FOUND) {
- LOG(INFO) << "path does not exist: " << remote_path;
- return Status::OK();
- } else if (list_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
- std::stringstream ss;
- ss << "failed to list files from remote path: " << remote_path
- << ", msg: " << list_rep.opStatus.message;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- LOG(INFO) << "finished to list files from remote path. file num: " << list_rep.files.size();
-
- // split file name and checksum
- for (const auto& file : list_rep.files) {
- if (file.isDir) {
- // this is not a file
- continue;
- }
-
- const std::string& file_name = file.path;
- size_t pos = file_name.find_last_of(".");
- if (pos == std::string::npos || pos == file_name.size() - 1) {
- // Not found checksum separator, ignore this file
- continue;
- }
-
- FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1),
- file.size};
- files->emplace(std::string(file_name, 0, pos), stat);
- VLOG(2) << "split remote file: " << std::string(file_name, 0, pos)
- << ", checksum: " << std::string(file_name, pos + 1);
- }
-
- LOG(INFO) << "finished to split files. valid file num: " << files->size();
-
- } catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "failed to list files in remote path: " << remote_path << ", msg: " << e.what();
- LOG(WARNING) << ss.str();
- return Status::RpcError(ss.str());
- }
-
- return status;
-}
-Status BrokerStorageBackend::direct_upload(const std::string& remote, const std::string& content) {
- std::vector<TNetworkAddress> broker_addrs;
- broker_addrs.push_back(_broker_addr);
- std::unique_ptr<BrokerWriter> broker_writer(
- new BrokerWriter(_env, broker_addrs, _broker_prop, remote, 0 /* offset */));
- RETURN_IF_ERROR(broker_writer->open());
- size_t write_len = 0;
- RETURN_IF_ERROR(broker_writer->write(reinterpret_cast<const uint8_t*>(content.c_str()),
- content.size(), &write_len));
- DCHECK_EQ(write_len, content.size());
- RETURN_IF_ERROR(broker_writer->close());
- return Status::OK();
-}
-
-Status BrokerStorageBackend::rm(const std::string& remote) {
- Status status = Status::OK();
- BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms,
- &status);
- if (!status.ok()) {
- std::stringstream ss;
- ss << "failed to get broker client. "
- << "broker addr: " << _broker_addr << ". msg: " << status;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- try {
- // rm file from remote path
- TBrokerDeletePathRequest del_req;
- TBrokerOperationStatus del_rep;
- del_req.__set_version(TBrokerVersion::VERSION_ONE);
- del_req.__set_path(remote);
- del_req.__set_properties(_broker_prop);
-
- try {
- client->deletePath(del_rep, del_req);
- } catch (apache::thrift::transport::TTransportException& e) {
- RETURN_IF_ERROR(client.reopen());
- client->deletePath(del_rep, del_req);
- }
-
- if (del_rep.statusCode == TBrokerOperationStatusCode::OK) {
- return Status::OK();
- } else {
- std::stringstream ss;
- ss << "failed to delete from remote path: " << remote << ", msg: " << del_rep.message;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- } catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "failed to delete file in remote path: " << remote << ", msg: " << e.what();
- LOG(WARNING) << ss.str();
- return Status::RpcError(ss.str());
- }
-}
-
-Status BrokerStorageBackend::rmdir(const std::string& remote) {
- return rm(remote);
-}
-
-Status BrokerStorageBackend::copy(const std::string& src, const std::string& dst) {
- return Status::NotSupported("copy not implemented!");
-}
-
-Status BrokerStorageBackend::copy_dir(const std::string& src, const std::string& dst) {
- return copy(src, dst);
-}
-
-Status BrokerStorageBackend::mkdir(const std::string& path) {
- return Status::NotSupported("mkdir not implemented!");
-}
-
-Status BrokerStorageBackend::mkdirs(const std::string& path) {
- return Status::NotSupported("mkdirs not implemented!");
-}
-
-Status BrokerStorageBackend::exist(const std::string& path) {
- Status status = Status::OK();
- BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms,
- &status);
- if (!status.ok()) {
- std::stringstream ss;
- ss << "failed to get broker client. "
- << "broker addr: " << _broker_addr << ". msg: " << status;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
- try {
- TBrokerCheckPathExistRequest check_req;
- TBrokerCheckPathExistResponse check_rep;
- check_req.__set_version(TBrokerVersion::VERSION_ONE);
- check_req.__set_path(path);
- check_req.__set_properties(_broker_prop);
-
- try {
- client->checkPathExist(check_rep, check_req);
- } catch (apache::thrift::transport::TTransportException& e) {
- RETURN_IF_ERROR(client.reopen());
- client->checkPathExist(check_rep, check_req);
- }
-
- if (check_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) {
- std::stringstream ss;
- ss << "failed to check exist: " << path << ", msg: " << check_rep.opStatus.message;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- } else if (!check_rep.isPathExist) {
- return Status::NotFound("{} not exists!", path);
- } else {
- return Status::OK();
- }
- } catch (apache::thrift::TException& e) {
- std::stringstream ss;
- ss << "failed to check exist: " << path << ", msg: " << e.what();
- LOG(WARNING) << ss.str();
- return Status::RpcError(ss.str());
- }
-}
-
-Status BrokerStorageBackend::exist_dir(const std::string& path) {
- return exist(path);
-}
-
-Status BrokerStorageBackend::upload_with_checksum(const std::string& local,
- const std::string& remote,
- const std::string& checksum) {
- std::string temp = remote + ".part";
- std::string final = remote + "." + checksum;
- RETURN_IF_ERROR(upload(local, remote + ".part"));
- return rename(temp, final);
-}
-
-} // end namespace doris
diff --git a/be/src/util/broker_storage_backend.h b/be/src/util/broker_storage_backend.h
deleted file mode 100644
index 09cdfdb436..0000000000
--- a/be/src/util/broker_storage_backend.h
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include "io/file_factory.h"
-#include "util/storage_backend.h"
-
-namespace doris {
-
-class ExecEnv;
-class TNetworkAddress;
-
-class BrokerStorageBackend : public StorageBackend {
-public:
- BrokerStorageBackend(ExecEnv* env, const TNetworkAddress& broker_addr,
- const std::map<std::string, std::string>& broker_prop);
- ~BrokerStorageBackend() {}
- Status download(const std::string& remote, const std::string& local) override;
- Status direct_download(const std::string& remote, std::string* content) override;
- Status upload(const std::string& local, const std::string& remote) override;
- Status upload_with_checksum(const std::string& local, const std::string& remote,
- const std::string& checksum) override;
- Status rename(const std::string& orig_name, const std::string& new_name) override;
- Status rename_dir(const std::string& orig_name, const std::string& new_name) override;
- Status list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) override;
- Status direct_upload(const std::string& remote, const std::string& content) override;
- Status rm(const std::string& remote) override;
- Status rmdir(const std::string& remote) override;
- Status copy(const std::string& src, const std::string& dst) override;
- Status copy_dir(const std::string& src, const std::string& dst) override;
- Status mkdir(const std::string& path) override;
- Status mkdirs(const std::string& path) override;
- Status exist(const std::string& path) override;
- Status exist_dir(const std::string& path) override;
-
-private:
- void _init_file_description(const std::string& remote);
-
- ExecEnv* _env;
- const TNetworkAddress& _broker_addr;
- const std::map<std::string, std::string>& _broker_prop;
- FileDescription _file_description;
-};
-} // end namespace doris
diff --git a/be/src/util/file_utils.cpp b/be/src/util/file_utils.cpp
index ae3b2837d0..d5c95caadd 100644
--- a/be/src/util/file_utils.cpp
+++ b/be/src/util/file_utils.cpp
@@ -33,7 +33,6 @@
#include "gutil/strings/split.h"
#include "gutil/strings/strip.h"
#include "gutil/strings/substitute.h"
-#include "olap/file_helper.h"
#include "runtime/thread_context.h"
#include "util/defer_op.h"
diff --git a/be/src/util/hdfs_storage_backend.cpp b/be/src/util/hdfs_storage_backend.cpp
deleted file mode 100644
index 6f021a4b73..0000000000
--- a/be/src/util/hdfs_storage_backend.cpp
+++ /dev/null
@@ -1,335 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/hdfs_storage_backend.h"
-
-#include "io/file_factory.h"
-#include "io/fs/file_reader.h"
-#include "io/fs/file_reader_options.h"
-#include "io/hdfs_writer.h"
-#include "olap/file_helper.h"
-#include "olap/iterators.h"
-#include "util/hdfs_util.h"
-
-namespace doris {
-
-#ifndef CHECK_HDFS_CLIENT
-#define CHECK_HDFS_CLIENT(client) \
- if (!client) { \
- return Status::InternalError("init hdfs client error."); \
- }
-#endif
-
-static const std::string hdfs_file_prefix = "hdfs://";
-
-HDFSStorageBackend::HDFSStorageBackend(const std::map<std::string, std::string>& prop)
- : _properties(prop) {
- HDFSCommonBuilder builder;
- Status st = createHDFSBuilder(_properties, &builder);
- if (st.ok()) {
- _hdfs_fs = HDFSHandle::instance().create_hdfs_fs(builder);
- DCHECK(_hdfs_fs) << "init hdfs client error.";
- }
- // if createHDFSBuilder failed, _hdfs_fs will be null.
- // and CHECK_HDFS_CLIENT will return error.
- // TODO: refacotr StorageBackend, unify into File system
-}
-
-HDFSStorageBackend::~HDFSStorageBackend() {
- close();
-}
-
-Status HDFSStorageBackend::close() {
- if (_hdfs_fs != nullptr) {
- hdfsDisconnect(_hdfs_fs);
- _hdfs_fs = nullptr;
- }
- return Status::OK();
-}
-
-// if the format of path is hdfs://ip:port/path, replace it to /path.
-// path like hdfs://ip:port/path can't be used by libhdfs3.
-std::string HDFSStorageBackend::parse_path(const std::string& path) {
- if (path.find(hdfs_file_prefix) != std::string::npos) {
- std::string temp = path.substr(hdfs_file_prefix.size());
- std::size_t pos = temp.find_first_of('/');
- return temp.substr(pos);
- } else {
- return path;
- }
-}
-
-Status HDFSStorageBackend::upload(const std::string& local, const std::string& remote) {
- FileHandler file_handler;
- Status ost = file_handler.open(local, O_RDONLY);
- if (!ost.ok()) {
- return Status::InternalError("failed to open file: " + local);
- }
-
- size_t file_len = file_handler.length();
- if (file_len == -1) {
- return Status::InternalError("failed to get length of file: " + local);
- }
-
- std::unique_ptr<HDFSWriter> hdfs_writer(new HDFSWriter(_properties, remote));
- RETURN_IF_ERROR(hdfs_writer->open());
- constexpr size_t buf_sz = 1024 * 1024;
- char read_buf[buf_sz];
- size_t left_len = file_len;
- size_t read_offset = 0;
- while (left_len > 0) {
- size_t read_len = left_len > buf_sz ? buf_sz : left_len;
- ost = file_handler.pread(read_buf, read_len, read_offset);
- if (!ost.ok()) {
- return Status::InternalError("failed to read file: " + local);
- }
-
- size_t write_len = 0;
- RETURN_IF_ERROR(hdfs_writer->write(reinterpret_cast<const uint8_t*>(read_buf), read_len,
- &write_len));
- DCHECK_EQ(write_len, read_len);
-
- read_offset += read_len;
- left_len -= read_len;
- }
-
- LOG(INFO) << "finished to write file: " << local << ", length: " << file_len;
- return Status::OK();
-}
-
-Status HDFSStorageBackend::direct_upload(const std::string& remote, const std::string& content) {
- std::unique_ptr<HDFSWriter> hdfs_writer(new HDFSWriter(_properties, remote));
- RETURN_IF_ERROR(hdfs_writer->open());
- size_t write_len = 0;
- RETURN_IF_ERROR(hdfs_writer->write(reinterpret_cast<const uint8_t*>(content.c_str()),
- content.size(), &write_len));
- DCHECK_EQ(write_len, content.size());
- return Status::OK();
-}
-
-Status HDFSStorageBackend::list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) {
- CHECK_HDFS_CLIENT(_hdfs_fs);
- std::string normal_str = parse_path(remote_path);
- int exists = hdfsExists(_hdfs_fs, normal_str.c_str());
- if (exists != 0) {
- LOG(INFO) << "path does not exist: " << normal_str << ", err: " << strerror(errno);
- return Status::OK();
- }
-
- int file_num = 0;
- hdfsFileInfo* files_info = hdfsListDirectory(_hdfs_fs, normal_str.c_str(), &file_num);
- if (files_info == nullptr) {
- std::stringstream ss;
- ss << "failed to list files from remote path: " << normal_str
- << ", err: " << strerror(errno);
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
-
- LOG(INFO) << "finished to list files from remote path:" << normal_str
- << ", file num:" << file_num;
- for (int i = 0; i < file_num; ++i) {
- auto file_info = files_info[i];
- if (file_info.mKind == kObjectKindDirectory) {
- continue;
- }
- const std::string& file_name_with_path(file_info.mName);
- // get filename
- std::filesystem::path file_path(file_name_with_path);
- std::string file_name = file_path.filename();
- size_t pos = file_name.find_last_of('.');
- if (pos == std::string::npos || pos == file_name.size() - 1) {
- // Not found checksum separator, ignore this file
- continue;
- }
-
- FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1),
- file_info.mSize};
- files->emplace(std::string(file_name, 0, pos), stat);
- VLOG(2) << "split remote file: " << std::string(file_name, 0, pos)
- << ", checksum: " << std::string(file_name, pos + 1);
- }
-
- hdfsFreeFileInfo(files_info, file_num);
- LOG(INFO) << "finished to split files. valid file num: " << files->size();
- return Status::OK();
-}
-
-Status HDFSStorageBackend::download(const std::string& remote, const std::string& local) {
- // 1. open remote file for read
- std::shared_ptr<io::FileSystem> file_system;
- io::FileReaderSPtr hdfs_reader = nullptr;
- auto cache_policy = io::FileCachePolicy::NO_CACHE;
- IOContext io_ctx;
- if (config::enable_file_cache && io_ctx.enable_file_cache) {
- cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE;
- }
- io::FileBlockCachePathPolicy file_block_cache;
- io::FileReaderOptions reader_options(cache_policy, file_block_cache);
- THdfsParams hdfs_params = parse_properties(_properties);
- RETURN_IF_ERROR(FileFactory::create_hdfs_reader(hdfs_params, remote, &file_system, &hdfs_reader,
- reader_options, &io_ctx));
-
- // 2. remove the existing local file if exist
- if (std::filesystem::remove(local)) {
- LOG(INFO) << "remove the previously exist local file: " << local;
- }
-
- // 3. open local file for write
- FileHandler file_handler;
- Status ost =
- file_handler.open_with_mode(local, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR);
- if (!ost.ok()) {
- return Status::InternalError("failed to open file: " + local);
- }
-
- // 4. read remote and write to local
- LOG(INFO) << "read remote file: " << remote << " to local: " << local;
- constexpr size_t buf_sz = 1024 * 1024;
- std::unique_ptr<char[]> read_buf(new char[buf_sz]);
- size_t write_offset = 0;
- size_t cur_offset = 0;
- while (true) {
- size_t read_len = 0;
- Slice file_slice(read_buf.get(), buf_sz);
- RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, io_ctx, &read_len));
- cur_offset += read_len;
- if (read_len == 0) {
- break;
- }
-
- ost = file_handler.pwrite(read_buf.get(), read_len, write_offset);
- if (!ost.ok()) {
- return Status::InternalError("failed to write file: " + local);
- }
- write_offset += read_len;
- }
-
- return Status::OK();
-}
-
-Status HDFSStorageBackend::direct_download(const std::string& remote, std::string* content) {
- std::shared_ptr<io::FileSystem> file_system;
- io::FileReaderSPtr hdfs_reader = nullptr;
- auto cache_policy = io::FileCachePolicy::NO_CACHE;
- IOContext io_ctx;
- if (config::enable_file_cache && io_ctx.enable_file_cache) {
- cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE;
- }
- io::FileBlockCachePathPolicy file_block_cache;
- io::FileReaderOptions reader_options(cache_policy, file_block_cache);
- THdfsParams hdfs_params = parse_properties(_properties);
- RETURN_IF_ERROR(FileFactory::create_hdfs_reader(hdfs_params, remote, &file_system, &hdfs_reader,
- reader_options, &io_ctx));
-
- constexpr size_t buf_sz = 1024 * 1024;
- std::unique_ptr<char[]> read_buf(new char[buf_sz]);
- size_t write_offset = 0;
- size_t cur_offset = 0;
- while (true) {
- size_t read_len = 0;
- Slice file_slice(read_buf.get(), buf_sz);
- RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, io_ctx, &read_len));
- cur_offset += read_len;
- if (read_len == 0) {
- break;
- }
-
- content->insert(write_offset, read_buf.get(), read_len);
- write_offset += read_len;
- }
- return Status::OK();
-}
-
-Status HDFSStorageBackend::upload_with_checksum(const std::string& local, const std::string& remote,
- const std::string& checksum) {
- std::string temp = remote + ".part";
- std::string final = remote + "." + checksum;
- RETURN_IF_ERROR(upload(local, temp));
- return rename(temp, final);
-}
-
-Status HDFSStorageBackend::rename(const std::string& orig_name, const std::string& new_name) {
- CHECK_HDFS_CLIENT(_hdfs_fs);
- std::string normal_orig_name = parse_path(orig_name);
- std::string normal_new_name = parse_path(new_name);
- int ret = hdfsRename(_hdfs_fs, normal_orig_name.c_str(), normal_new_name.c_str());
- if (ret == 0) {
- LOG(INFO) << "finished to rename file. orig: " << normal_orig_name
- << ", new: " << normal_new_name;
- return Status::OK();
- } else {
- std::stringstream ss;
- ss << "Fail to rename file: " << normal_orig_name << " to: " << normal_new_name;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
-}
-
-Status HDFSStorageBackend::rename_dir(const std::string& orig_name, const std::string& new_name) {
- return rename(orig_name, new_name);
-}
-
-Status HDFSStorageBackend::copy(const std::string& src, const std::string& dst) {
- return Status::NotSupported("copy not implemented!");
-}
-
-Status HDFSStorageBackend::copy_dir(const std::string& src, const std::string& dst) {
- return copy(src, dst);
-}
-
-Status HDFSStorageBackend::mkdir(const std::string& path) {
- return Status::NotSupported("mkdir not implemented!");
-}
-
-Status HDFSStorageBackend::mkdirs(const std::string& path) {
- return Status::NotSupported("mkdirs not implemented!");
-}
-
-Status HDFSStorageBackend::exist(const std::string& path) {
- CHECK_HDFS_CLIENT(_hdfs_fs);
- int exists = hdfsExists(_hdfs_fs, path.c_str());
- if (exists == 0) {
- return Status::OK();
- } else {
- return Status::NotFound(path + " not exists!");
- }
-}
-
-Status HDFSStorageBackend::exist_dir(const std::string& path) {
- return exist(path);
-}
-
-Status HDFSStorageBackend::rm(const std::string& remote) {
- CHECK_HDFS_CLIENT(_hdfs_fs);
- int ret = hdfsDelete(_hdfs_fs, remote.c_str(), 1);
- if (ret == 0) {
- return Status::OK();
- } else {
- std::stringstream ss;
- ss << "failed to delete from remote path: " << remote;
- LOG(WARNING) << ss.str();
- return Status::InternalError(ss.str());
- }
-}
-
-Status HDFSStorageBackend::rmdir(const std::string& remote) {
- return rm(remote);
-}
-
-} // end namespace doris
diff --git a/be/src/util/hdfs_storage_backend.h b/be/src/util/hdfs_storage_backend.h
deleted file mode 100644
index acbf18d2d0..0000000000
--- a/be/src/util/hdfs_storage_backend.h
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <hdfs/hdfs.h>
-
-#include "io/hdfs_builder.h"
-#include "util/storage_backend.h"
-
-namespace doris {
-
-class HDFSStorageBackend : public StorageBackend {
-public:
- HDFSStorageBackend(const std::map<std::string, std::string>& prop);
- ~HDFSStorageBackend();
-
- Status close();
- Status download(const std::string& remote, const std::string& local) override;
- Status direct_download(const std::string& remote, std::string* content) override;
- Status upload(const std::string& local, const std::string& remote) override;
- Status upload_with_checksum(const std::string& local, const std::string& remote,
- const std::string& checksum) override;
- Status list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) override;
- Status rename(const std::string& orig_name, const std::string& new_name) override;
- Status rename_dir(const std::string& orig_name, const std::string& new_name) override;
- Status direct_upload(const std::string& remote, const std::string& content) override;
- Status rm(const std::string& remote) override;
- Status rmdir(const std::string& remote) override;
- Status copy(const std::string& src, const std::string& dst) override;
- Status copy_dir(const std::string& src, const std::string& dst) override;
- Status mkdir(const std::string& path) override;
- Status mkdirs(const std::string& path) override;
- Status exist(const std::string& path) override;
- Status exist_dir(const std::string& path) override;
-
-private:
- // if the format of path is hdfs://ip:port/path, replace it to /path.
- // path like hdfs://ip:port/path can't be used by libhdfs3.
- std::string parse_path(const std::string& path);
-
-private:
- std::map<std::string, std::string> _properties;
- hdfsFS _hdfs_fs = nullptr;
-};
-
-} // end namespace doris
diff --git a/be/src/util/hdfs_util.cpp b/be/src/util/hdfs_util.cpp
index b58fc75e46..a5ad09ec01 100644
--- a/be/src/util/hdfs_util.cpp
+++ b/be/src/util/hdfs_util.cpp
@@ -23,6 +23,7 @@
#include "common/logging.h"
namespace doris {
+namespace io {
HDFSHandle& HDFSHandle::instance() {
static HDFSHandle hdfs_handle;
@@ -39,4 +40,14 @@ hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) {
return hdfs_fs;
}
+Path convert_path(const Path& path, const std::string& namenode) {
+ Path real_path(path);
+ if (path.string().find(namenode) != std::string::npos) {
+ std::string real_path_str = path.string().substr(namenode.size());
+ real_path = real_path_str;
+ }
+ return real_path;
+}
+
+} // namespace io
} // namespace doris
diff --git a/be/src/util/hdfs_util.h b/be/src/util/hdfs_util.h
index f7bfc14b3a..f98bdd5ab3 100644
--- a/be/src/util/hdfs_util.h
+++ b/be/src/util/hdfs_util.h
@@ -24,9 +24,11 @@
#include <string>
#include "common/status.h"
+#include "io/fs/path.h"
#include "io/hdfs_builder.h"
namespace doris {
+namespace io {
class HDFSHandle {
public:
@@ -40,4 +42,9 @@ private:
HDFSHandle() {}
};
-} // namespace doris
\ No newline at end of file
+// if the format of path is hdfs://ip:port/path, replace it to /path.
+// path like hdfs://ip:port/path can't be used by libhdfs3.
+Path convert_path(const Path& path, const std::string& namenode);
+
+} // namespace io
+} // namespace doris
diff --git a/be/src/util/s3_storage_backend.cpp b/be/src/util/s3_storage_backend.cpp
deleted file mode 100644
index f074e682c1..0000000000
--- a/be/src/util/s3_storage_backend.cpp
+++ /dev/null
@@ -1,307 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/s3_storage_backend.h"
-
-#include <aws/s3/S3Client.h>
-#include <aws/s3/model/CopyObjectRequest.h>
-#include <aws/s3/model/DeleteObjectRequest.h>
-#include <aws/s3/model/GetObjectRequest.h>
-#include <aws/s3/model/HeadObjectRequest.h>
-#include <aws/s3/model/ListObjectsRequest.h>
-#include <aws/s3/model/PutObjectRequest.h>
-
-#include <boost/algorithm/string.hpp>
-#include <filesystem>
-#include <fstream>
-#include <iostream>
-#include <sstream>
-
-#include "common/logging.h"
-#include "gutil/strings/strip.h"
-#include "util/s3_uri.h"
-#include "util/s3_util.h"
-
-namespace doris {
-
-#ifndef CHECK_S3_CLIENT
-#define CHECK_S3_CLIENT(client) \
- if (!client) { \
- return Status::InternalError("init aws s3 client error."); \
- }
-#endif
-
-#ifndef CHECK_S3_PATH
-#define CHECK_S3_PATH(uri, path) \
- S3URI uri(path); \
- if (!uri.parse()) { \
- return Status::InvalidArgument("s3 uri is invalid: {}", path); \
- }
-#endif
-
-#ifndef RETRUN_S3_STATUS
-#define RETRUN_S3_STATUS(response) \
- if (response.IsSuccess()) { \
- return Status::OK(); \
- } else { \
- return Status::InternalError(error_msg(response)); \
- }
-#endif
-
-S3StorageBackend::S3StorageBackend(const std::map<std::string, std::string>& prop)
- : _properties(prop) {
- _client = ClientFactory::instance().create(_properties);
- DCHECK(_client) << "init aws s3 client error.";
-}
-
-S3StorageBackend::~S3StorageBackend() {}
-
-Status S3StorageBackend::download(const std::string& remote, const std::string& local) {
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, remote);
- Aws::S3::Model::GetObjectRequest request;
- request.WithBucket(uri.get_bucket()).WithKey(uri.get_key());
- Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request);
- if (response.IsSuccess()) {
- Aws::OFStream local_file;
- local_file.open(local, std::ios::out | std::ios::binary);
- if (local_file.good()) {
- local_file << response.GetResult().GetBody().rdbuf();
- }
- if (!local_file.good()) {
- return Status::InternalError("failed to write file: {}", local);
- }
- } else {
- return Status::IOError("s3 download error: {}", error_msg(response));
- }
- return Status::OK();
-}
-
-Status S3StorageBackend::direct_download(const std::string& remote, std::string* content) {
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, remote);
- Aws::S3::Model::GetObjectRequest request;
- request.WithBucket(uri.get_bucket()).WithKey(uri.get_key());
- Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request);
- if (response.IsSuccess()) {
- std::stringstream ss;
- ss << response.GetResult().GetBody().rdbuf();
- *content = ss.str();
- } else {
- return Status::IOError("s3 direct_download error: {}", error_msg(response));
- }
- return Status::OK();
-}
-
-Status S3StorageBackend::upload(const std::string& local, const std::string& remote) {
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, remote);
- Aws::S3::Model::PutObjectRequest request;
- request.WithBucket(uri.get_bucket()).WithKey(uri.get_key());
-
- const std::shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::FStream>(
- local.c_str(), local.c_str(), std::ios_base::in | std::ios_base::binary);
- if (input_data->good()) {
- request.SetBody(input_data);
- }
- if (!input_data->good()) {
- return Status::InternalError("failed to read file: {}", local);
- }
- Aws::S3::Model::PutObjectOutcome response = _client->PutObject(request);
-
- RETRUN_S3_STATUS(response);
-}
-
-Status S3StorageBackend::list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) {
- std::string normal_str(remote_path);
- if (!normal_str.empty() && normal_str.at(normal_str.size() - 1) != '/') {
- normal_str += '/';
- }
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, normal_str);
-
- Aws::S3::Model::ListObjectsRequest request;
- request.WithBucket(uri.get_bucket()).WithPrefix(uri.get_key());
- if (!recursion) {
- request.WithDelimiter("/");
- }
- Aws::S3::Model::ListObjectsOutcome response = _client->ListObjects(request);
- if (response.IsSuccess()) {
- Aws::Vector<Aws::S3::Model::Object> objects = response.GetResult().GetContents();
-
- for (Aws::S3::Model::Object& object : objects) {
- std::string key = object.GetKey();
- if (key.at(key.size() - 1) == '/') {
- continue;
- }
- key = boost::algorithm::replace_first_copy(key, uri.get_key(), "");
- if (contain_md5) {
- size_t pos = key.find_last_of(".");
- if (pos == std::string::npos || pos == key.size() - 1) {
- // Not found checksum separator, ignore this file
- continue;
- }
- FileStat stat = {std::string(key, 0, pos), std::string(key, pos + 1),
- object.GetSize()};
- files->emplace(std::string(key, 0, pos), stat);
- } else {
- FileStat stat = {key, "", object.GetSize()};
- files->emplace(key, stat);
- }
- }
- return Status::OK();
- } else {
- return Status::InternalError("list form s3 error: {}", error_msg(response));
- }
-}
-
-Status S3StorageBackend::rename(const std::string& orig_name, const std::string& new_name) {
- RETURN_IF_ERROR(copy(orig_name, new_name));
- return rm(orig_name);
-}
-
-Status S3StorageBackend::rename_dir(const std::string& orig_name, const std::string& new_name) {
- RETURN_IF_ERROR(copy_dir(orig_name, new_name));
- return rmdir(orig_name);
-}
-
-Status S3StorageBackend::direct_upload(const std::string& remote, const std::string& content) {
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, remote);
- Aws::S3::Model::PutObjectRequest request;
- request.WithBucket(uri.get_bucket()).WithKey(uri.get_key());
- const std::shared_ptr<Aws::IOStream> input_data =
- Aws::MakeShared<Aws::StringStream>("upload_directly");
- *input_data << content.c_str();
- if (input_data->good()) {
- request.SetBody(input_data);
- }
- if (!input_data->good()) {
- return Status::InternalError("failed to read from string");
- }
- Aws::S3::Model::PutObjectOutcome response = _client->PutObject(request);
-
- RETRUN_S3_STATUS(response);
-}
-
-Status S3StorageBackend::rm(const std::string& remote) {
- LOG(INFO) << "S3 rm file: " << remote;
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, remote);
- Aws::S3::Model::DeleteObjectRequest request;
-
- request.WithKey(uri.get_key()).WithBucket(uri.get_bucket());
-
- Aws::S3::Model::DeleteObjectOutcome response = _client->DeleteObject(request);
-
- RETRUN_S3_STATUS(response);
-}
-
-Status S3StorageBackend::rmdir(const std::string& remote) {
- CHECK_S3_CLIENT(_client);
- std::map<std::string, FileStat> files;
- std::string normal_path(remote);
- if (!normal_path.empty() && normal_path.at(normal_path.size() - 1) != '/') {
- normal_path += '/';
- }
- LOG(INFO) << "Remove S3 dir: " << remote;
- RETURN_IF_ERROR(list(normal_path, false, true, &files));
-
- for (auto& file : files) {
- std::string file_path = normal_path + file.second.name;
- RETURN_IF_ERROR(rm(file_path));
- }
- return Status::OK();
-}
-
-Status S3StorageBackend::copy(const std::string& src, const std::string& dst) {
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(src_uri, src);
- CHECK_S3_PATH(dst_uri, dst);
- Aws::S3::Model::CopyObjectRequest request;
- request.WithCopySource(src_uri.get_bucket() + "/" + src_uri.get_key())
- .WithKey(dst_uri.get_key())
- .WithBucket(dst_uri.get_bucket());
-
- Aws::S3::Model::CopyObjectOutcome response = _client->CopyObject(request);
-
- RETRUN_S3_STATUS(response);
-}
-
-Status S3StorageBackend::copy_dir(const std::string& src, const std::string& dst) {
- std::map<std::string, FileStat> files;
- LOG(INFO) << "Copy S3 dir: " << src << " -> " << dst;
- RETURN_IF_ERROR(list(src, false, true, &files));
- if (files.size() == 0) {
- LOG(WARNING) << "Nothing need to copy: " << src << " -> " << dst;
- return Status::OK();
- }
- for (auto& kv : files) {
- RETURN_IF_ERROR(copy(src + "/" + kv.first, dst + "/" + kv.first));
- }
- return Status::OK();
-}
-
-// dir is not supported by s3, it will cause something confusion.
-Status S3StorageBackend::mkdir(const std::string& path) {
- return Status::OK();
-}
-
-// dir is not supported by s3, it will cause something confusion.
-Status S3StorageBackend::mkdirs(const std::string& path) {
- return Status::OK();
-}
-
-Status S3StorageBackend::exist(const std::string& path) {
- CHECK_S3_CLIENT(_client);
- CHECK_S3_PATH(uri, path);
- Aws::S3::Model::HeadObjectRequest request;
- request.WithBucket(uri.get_bucket()).WithKey(uri.get_key());
- Aws::S3::Model::HeadObjectOutcome response = _client->HeadObject(request);
- if (response.IsSuccess()) {
- return Status::OK();
- } else if (response.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) {
- return Status::NotFound("{} not exists!", path);
- } else {
- return Status::InternalError(error_msg(response));
- }
-}
-
-Status S3StorageBackend::exist_dir(const std::string& path) {
- std::map<std::string, FileStat> files;
- RETURN_IF_ERROR(list(path, false, true, &files));
- if (files.size() > 0) {
- return Status::OK();
- }
- return Status::NotFound("{} not exists!", path);
-}
-
-Status S3StorageBackend::upload_with_checksum(const std::string& local, const std::string& remote,
- const std::string& checksum) {
- return upload(local, remote + "." + checksum);
-}
-
-template <typename AwsOutcome>
-std::string S3StorageBackend::error_msg(const AwsOutcome& outcome) {
- std::stringstream out;
- out << "Error: [" << outcome.GetError().GetExceptionName() << ":"
- << outcome.GetError().GetMessage();
- return out.str();
-}
-
-} // end namespace doris
diff --git a/be/src/util/s3_storage_backend.h b/be/src/util/s3_storage_backend.h
deleted file mode 100644
index b6d31c5593..0000000000
--- a/be/src/util/s3_storage_backend.h
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <util/storage_backend.h>
-
-namespace Aws {
-namespace S3 {
-class S3Client;
-} // namespace S3
-} // namespace Aws
-
-namespace doris {
-
-class S3StorageBackend : public StorageBackend {
-public:
- S3StorageBackend(const std::map<std::string, std::string>& prop);
- ~S3StorageBackend();
- Status download(const std::string& remote, const std::string& local) override;
- Status direct_download(const std::string& remote, std::string* content) override;
- Status upload(const std::string& local, const std::string& remote) override;
- Status upload_with_checksum(const std::string& local, const std::string& remote,
- const std::string& checksum) override;
- Status list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) override;
- Status rename(const std::string& orig_name, const std::string& new_name) override;
- Status rename_dir(const std::string& orig_name, const std::string& new_name) override;
- Status direct_upload(const std::string& remote, const std::string& content) override;
- Status rm(const std::string& remote) override;
- Status rmdir(const std::string& remote) override;
- Status copy(const std::string& src, const std::string& dst) override;
- Status copy_dir(const std::string& src, const std::string& dst) override;
- Status mkdir(const std::string& path) override;
- Status mkdirs(const std::string& path) override;
- Status exist(const std::string& path) override;
- Status exist_dir(const std::string& path) override;
-
-private:
- template <typename AwsOutcome>
- std::string error_msg(const AwsOutcome& outcome);
- const std::map<std::string, std::string>& _properties;
- std::shared_ptr<Aws::S3::S3Client> _client;
-};
-
-} // end namespace doris
diff --git a/be/src/util/s3_uri.cpp b/be/src/util/s3_uri.cpp
index 36ceec0eae..bbc664237c 100644
--- a/be/src/util/s3_uri.cpp
+++ b/be/src/util/s3_uri.cpp
@@ -29,39 +29,42 @@ const std::string S3URI::_SCHEME_DELIM = "://";
const std::string S3URI::_PATH_DELIM = "/";
const std::string S3URI::_QUERY_DELIM = "?";
const std::string S3URI::_FRAGMENT_DELIM = "#";
-const StringCaseSet S3URI::_VALID_SCHEMES = {"http", "https", "s3", "s3a", "s3n", "bos", "oss"};
-bool S3URI::parse() {
+const StringCaseSet S3URI::_VALID_SCHEMES = {"http", "https", "s3", "s3a", "s3n",
+ "bos", "oss", "cos", "obs"};
+
+/// eg:
+/// s3://bucket1/path/to/file.txt
+/// _schema: s3
+/// _bucket: bucket1
+/// _key: path/to/file.txt
+Status S3URI::parse() {
if (_location.empty()) {
- return false;
+ return Status::InvalidArgument("location is empty");
}
std::vector<std::string> scheme_split = strings::Split(_location, _SCHEME_DELIM);
if (scheme_split.size() != 2) {
- LOG(WARNING) << "Invalid S3 URI: " << _location;
- return false;
+ return Status::InvalidArgument("Invalid S3 URI: {}", _location);
}
_scheme = scheme_split[0];
if (_VALID_SCHEMES.find(_scheme) == _VALID_SCHEMES.end()) {
- LOG(WARNING) << "Invalid scheme: " << _scheme;
- return false;
+ return Status::InvalidArgument("Invalid scheme: {}", _scheme);
}
std::vector<std::string> authority_split =
strings::Split(scheme_split[1], strings::delimiter::Limit(_PATH_DELIM, 1));
if (authority_split.size() != 2) {
- LOG(WARNING) << "Invalid S3 URI: " << _location;
- return false;
+ return Status::InvalidArgument("Invalid S3 URI: {}", _location);
}
_key = authority_split[1];
StripWhiteSpace(&_key);
if (_key.empty()) {
- LOG(WARNING) << "Invalid S3 key: " << _location;
- return false;
+ return Status::InvalidArgument("Invalid S3 key: {}", _location);
}
_bucket = authority_split[0];
// Strip query and fragment if they exist
std::vector<std::string> _query_split = strings::Split(_key, _QUERY_DELIM);
std::vector<std::string> _fragment_split = strings::Split(_query_split[0], _FRAGMENT_DELIM);
_key = _fragment_split[0];
- return true;
+ return Status::OK();
}
std::string S3URI::to_string() const {
diff --git a/be/src/util/s3_uri.h b/be/src/util/s3_uri.h
index 46e61b3282..de4bd05547 100644
--- a/be/src/util/s3_uri.h
+++ b/be/src/util/s3_uri.h
@@ -19,6 +19,7 @@
#include <string>
+#include "common/status.h"
#include "util/string_util.h"
namespace doris {
@@ -26,7 +27,7 @@ namespace doris {
class S3URI {
public:
S3URI(const std::string& location) : _location(location) {}
- bool parse();
+ Status parse();
const std::string& get_bucket() const { return _bucket; }
const std::string& get_key() const { return _key; }
const std::string& get_location() const { return _location; }
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 7b1c88b220..c797dcb203 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -77,7 +77,7 @@ private:
const static std::string USE_PATH_STYLE = "use_path_style";
-ClientFactory::ClientFactory() {
+S3ClientFactory::S3ClientFactory() {
_aws_options = Aws::SDKOptions {};
Aws::Utils::Logging::LogLevel logLevel =
static_cast<Aws::Utils::Logging::LogLevel>(config::aws_log_level);
@@ -88,16 +88,16 @@ ClientFactory::ClientFactory() {
Aws::InitAPI(_aws_options);
}
-ClientFactory::~ClientFactory() {
+S3ClientFactory::~S3ClientFactory() {
Aws::ShutdownAPI(_aws_options);
}
-ClientFactory& ClientFactory::instance() {
- static ClientFactory ret;
+S3ClientFactory& S3ClientFactory::instance() {
+ static S3ClientFactory ret;
return ret;
}
-bool ClientFactory::is_s3_conf_valid(const std::map<std::string, std::string>& prop) {
+bool S3ClientFactory::is_s3_conf_valid(const std::map<std::string, std::string>& prop) {
StringCaseMap<std::string> properties(prop.begin(), prop.end());
if (properties.find(S3_AK) == properties.end() || properties.find(S3_SK) == properties.end() ||
properties.find(S3_ENDPOINT) == properties.end() ||
@@ -109,7 +109,7 @@ bool ClientFactory::is_s3_conf_valid(const std::map<std::string, std::string>& p
return true;
}
-std::shared_ptr<Aws::S3::S3Client> ClientFactory::create(
+std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(
const std::map<std::string, std::string>& prop) {
if (!is_s3_conf_valid(prop)) {
return nullptr;
@@ -148,11 +148,11 @@ std::shared_ptr<Aws::S3::S3Client> ClientFactory::create(
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, use_virtual_addressing);
}
-bool ClientFactory::is_s3_conf_valid(const S3Conf& s3_conf) {
+bool S3ClientFactory::is_s3_conf_valid(const S3Conf& s3_conf) {
return !s3_conf.ak.empty() && !s3_conf.sk.empty() && !s3_conf.endpoint.empty();
}
-std::shared_ptr<Aws::S3::S3Client> ClientFactory::create(const S3Conf& s3_conf) {
+std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(const S3Conf& s3_conf) {
if (!is_s3_conf_valid(s3_conf)) {
return nullptr;
}
@@ -177,8 +177,8 @@ std::shared_ptr<Aws::S3::S3Client> ClientFactory::create(const S3Conf& s3_conf)
s3_conf.use_virtual_addressing);
}
-Status ClientFactory::convert_properties_to_s3_conf(const std::map<std::string, std::string>& prop,
- const S3URI& s3_uri, S3Conf* s3_conf) {
+Status S3ClientFactory::convert_properties_to_s3_conf(
+ const std::map<std::string, std::string>& prop, const S3URI& s3_uri, S3Conf* s3_conf) {
if (!is_s3_conf_valid(prop)) {
return Status::InvalidArgument("S3 properties are incorrect, please check properties.");
}
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 4adcc72874..7ef5ca00c5 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -65,11 +65,11 @@ struct S3Conf {
}
};
-class ClientFactory {
+class S3ClientFactory {
public:
- ~ClientFactory();
+ ~S3ClientFactory();
- static ClientFactory& instance();
+ static S3ClientFactory& instance();
std::shared_ptr<Aws::S3::S3Client> create(const std::map<std::string, std::string>& prop);
@@ -83,7 +83,7 @@ public:
const S3URI& s3_uri, S3Conf* s3_conf);
private:
- ClientFactory();
+ S3ClientFactory();
Aws::SDKOptions _aws_options;
};
diff --git a/be/src/util/storage_backend.h b/be/src/util/storage_backend.h
deleted file mode 100644
index d18b4e73b0..0000000000
--- a/be/src/util/storage_backend.h
+++ /dev/null
@@ -1,53 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include "common/status.h"
-
-namespace doris {
-
-struct FileStat {
- std::string name;
- std::string md5;
- int64_t size;
-};
-
-class StorageBackend {
-public:
- virtual Status download(const std::string& remote, const std::string& local) = 0;
- virtual Status direct_download(const std::string& remote, std::string* content) = 0;
- virtual Status upload(const std::string& local, const std::string& remote) = 0;
- virtual Status upload_with_checksum(const std::string& local, const std::string& remote,
- const std::string& checksum) = 0;
- virtual Status list(const std::string& remote_path, bool contain_md5, bool recursion,
- std::map<std::string, FileStat>* files) = 0;
- virtual Status rename(const std::string& orig_name, const std::string& new_name) = 0;
- virtual Status rename_dir(const std::string& orig_name, const std::string& new_name) = 0;
- virtual Status direct_upload(const std::string& remote, const std::string& content) = 0;
- virtual Status copy(const std::string& src, const std::string& dst) = 0;
- virtual Status copy_dir(const std::string& src, const std::string& dst) = 0;
- virtual Status rm(const std::string& remote) = 0;
- virtual Status rmdir(const std::string& remote) = 0;
- virtual Status mkdir(const std::string& path) = 0;
- virtual Status mkdirs(const std::string& path) = 0;
- virtual Status exist(const std::string& path) = 0;
- virtual Status exist_dir(const std::string& path) = 0;
-
- virtual ~StorageBackend() = default;
-};
-} // end namespace doris
diff --git a/be/src/util/storage_backend_mgr.cpp b/be/src/util/storage_backend_mgr.cpp
deleted file mode 100644
index 4d940faf05..0000000000
--- a/be/src/util/storage_backend_mgr.cpp
+++ /dev/null
@@ -1,217 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/storage_backend_mgr.h"
-
-#include "common/config.h"
-#include "common/status.h"
-#include "env/env.h"
-#include "env/env_util.h"
-#include "gutil/strings/substitute.h"
-#include "util/file_utils.h"
-#include "util/s3_storage_backend.h"
-#include "util/s3_util.h"
-#include "util/storage_backend.h"
-
-namespace doris {
-using namespace ErrorCode;
-
-Status StorageBackendMgr::init(const std::string& storage_param_dir) {
- if (_is_inited) {
- return Status::OK();
- }
- Status exist_status = Env::Default()->path_exists(storage_param_dir);
- if (!exist_status.ok() &&
- (!exist_status.is<NOT_FOUND>() || !Env::Default()->create_dirs(storage_param_dir).ok())) {
- RETURN_NOT_OK_STATUS_WITH_WARN(
- Status::IOError("failed to create remote storage_param root path {}",
- storage_param_dir),
- "create_dirs failed");
- }
-
- std::vector<std::string> file_names;
- RETURN_IF_ERROR(FileUtils::list_files(Env::Default(), storage_param_dir, &file_names));
- for (auto& file_name : file_names) {
- std::string buf;
- RETURN_NOT_OK_STATUS_WITH_WARN(
- env_util::read_file_to_string(Env::Default(), storage_param_dir + "/" + file_name,
- &buf),
- strings::Substitute("load storage_name failed. $0", file_name));
- StorageParamPB storage_param_pb;
- RETURN_IF_ERROR(_deserialize_param(buf, &storage_param_pb));
- RETURN_IF_ERROR(_create_remote_storage_internal(storage_param_pb));
- LOG(INFO) << "init remote_storage_param successfully. storage_name: " << file_name;
- }
- _storage_param_dir = storage_param_dir;
- _is_inited = true;
- return Status::OK();
-}
-
-Status StorageBackendMgr::create_remote_storage(const StorageParamPB& storage_param_pb) {
- if (_check_exist(storage_param_pb)) {
- return Status::OK();
- }
- RETURN_IF_ERROR(_create_remote_storage_internal(storage_param_pb));
-
- std::string storage_name = storage_param_pb.storage_name();
-
- string storage_param_path = _storage_param_dir + "/" + storage_name;
- RETURN_NOT_OK_STATUS_WITH_WARN(
- FileUtils::remove(storage_param_path),
- strings::Substitute("rm storage_param_pb file failed: $0", storage_param_path));
- std::string param_binary;
- RETURN_NOT_OK_STATUS_WITH_WARN(_serialize_param(storage_param_pb, ¶m_binary),
- "_serialize_param storage_param_pb failed.");
- RETURN_NOT_OK_STATUS_WITH_WARN(
- env_util::write_string_to_file(Env::Default(), Slice(param_binary), storage_param_path),
- strings::Substitute("write_string_to_file failed: $0", storage_param_path));
- std::string buf;
- RETURN_NOT_OK_STATUS_WITH_WARN(
- env_util::read_file_to_string(Env::Default(), storage_param_path, &buf),
- strings::Substitute("read storage_name failed. $0", storage_param_path));
- if (buf != param_binary) {
- LOG(ERROR) << "storage_param written failed. storage_name: ("
- << storage_param_pb.storage_name() << "<->" << storage_name << ")";
- return Status::InternalError("storage_param written failed");
- }
- LOG(INFO) << "create remote_storage_param successfully. storage_name: " << storage_name;
- return Status::OK();
-}
-
-Status StorageBackendMgr::_create_remote_storage_internal(const StorageParamPB& storage_param_pb) {
- std::string storage_name = storage_param_pb.storage_name();
- std::unique_lock wrlock(_storage_backend_lock);
- if (_storage_backend_map.size() >= doris::config::max_remote_storage_count) {
- std::map<std::string, time_t>::iterator itr = _storage_backend_active_time.begin();
- std::string timeout_key = itr->first;
- time_t min_active_time = itr->second;
- ++itr;
- for (; itr != _storage_backend_active_time.end(); ++itr) {
- if (itr->second < min_active_time) {
- timeout_key = itr->first;
- min_active_time = itr->second;
- }
- }
- _storage_backend_map.erase(storage_name);
- _storage_param_map.erase(storage_name);
- _storage_backend_active_time.erase(storage_name);
- }
- std::map<std::string, std::string> storage_prop;
- switch (storage_param_pb.storage_medium()) {
- case StorageMediumPB::S3:
- default:
- S3StorageParamPB s3_storage_param = storage_param_pb.s3_storage_param();
- if (s3_storage_param.s3_ak().empty() || s3_storage_param.s3_sk().empty() ||
- s3_storage_param.s3_endpoint().empty() || s3_storage_param.s3_region().empty()) {
- return Status::InternalError("s3_storage_param param is invalid");
- }
- storage_prop[S3_AK] = s3_storage_param.s3_ak();
- storage_prop[S3_SK] = s3_storage_param.s3_sk();
- storage_prop[S3_ENDPOINT] = s3_storage_param.s3_endpoint();
- storage_prop[S3_REGION] = s3_storage_param.s3_region();
- storage_prop[S3_MAX_CONN_SIZE] = s3_storage_param.s3_max_conn();
- storage_prop[S3_REQUEST_TIMEOUT_MS] = s3_storage_param.s3_request_timeout_ms();
- storage_prop[S3_CONN_TIMEOUT_MS] = s3_storage_param.s3_conn_timeout_ms();
-
- if (!ClientFactory::is_s3_conf_valid(storage_prop)) {
- return Status::InternalError("s3_storage_param is invalid");
- }
- _storage_backend_map[storage_name] = std::make_shared<S3StorageBackend>(storage_prop);
- }
- _storage_param_map[storage_name] = storage_param_pb;
- _storage_backend_active_time[storage_name] = time(nullptr);
-
- return Status::OK();
-}
-
-std::shared_ptr<StorageBackend> StorageBackendMgr::get_storage_backend(
- const std::string& storage_name) {
- std::shared_lock rdlock(_storage_backend_lock);
- if (_storage_backend_map.find(storage_name) == _storage_backend_map.end()) {
- return nullptr;
- }
- _storage_backend_active_time[storage_name] = time(nullptr);
- return _storage_backend_map[storage_name];
-}
-
-Status StorageBackendMgr::get_storage_param(const std::string& storage_name,
- StorageParamPB* storage_param) {
- std::shared_lock rdlock(_storage_backend_lock);
- if (_storage_backend_map.find(storage_name) == _storage_backend_map.end()) {
- return Status::InternalError("storage_name not exist: {}", storage_name);
- }
- *storage_param = _storage_param_map[storage_name];
- return Status::OK();
-}
-
-Status StorageBackendMgr::get_root_path(const std::string& storage_name, std::string* root_path) {
- std::shared_lock rdlock(_storage_backend_lock);
- if (_storage_backend_map.find(storage_name) == _storage_backend_map.end()) {
- return Status::InternalError("storage_name not exist: {}", storage_name);
- }
- *root_path = get_root_path_from_param(_storage_param_map[storage_name]);
- return Status::OK();
-}
-
-std::string StorageBackendMgr::get_root_path_from_param(const StorageParamPB& storage_param) {
- switch (storage_param.storage_medium()) {
- case StorageMediumPB::S3:
- default: {
- return storage_param.s3_storage_param().root_path();
- }
- }
-}
-
-Status StorageBackendMgr::_check_exist(const StorageParamPB& storage_param_pb) {
- StorageParamPB old_storage_param;
- RETURN_IF_ERROR(get_storage_param(storage_param_pb.storage_name(), &old_storage_param));
- std::shared_lock rdlock(_storage_backend_lock);
- std::string old_param_binary;
- RETURN_NOT_OK_STATUS_WITH_WARN(_serialize_param(old_storage_param, &old_param_binary),
- "_serialize_param old_storage_param_pb failed.");
- std::string param_binary;
- RETURN_NOT_OK_STATUS_WITH_WARN(_serialize_param(storage_param_pb, ¶m_binary),
- "_serialize_param storage_param_pb failed.");
- if (old_param_binary != param_binary) {
- LOG(ERROR) << "storage_param has been changed: " << storage_param_pb.storage_name();
- return Status::InternalError("storage_param has been changed");
- }
- return Status::OK();
-}
-
-Status StorageBackendMgr::_serialize_param(const StorageParamPB& storage_param_pb,
- std::string* param_binary) {
- bool serialize_success = storage_param_pb.SerializeToString(param_binary);
- if (!serialize_success) {
- LOG(WARNING) << "failed to serialize storage_param " << storage_param_pb.storage_name();
- return Status::InternalError("failed to serialize storage_param: {}",
- storage_param_pb.storage_name());
- }
- return Status::OK();
-}
-
-Status StorageBackendMgr::_deserialize_param(const std::string& param_binary,
- StorageParamPB* storage_param_pb) {
- bool parsed = storage_param_pb->ParseFromString(param_binary);
- if (!parsed) {
- LOG(WARNING) << "parse storage_param failed";
- return Status::InternalError("parse storage_param failed");
- }
- return Status::OK();
-}
-
-} // namespace doris
\ No newline at end of file
diff --git a/be/src/util/storage_backend_mgr.h b/be/src/util/storage_backend_mgr.h
deleted file mode 100644
index 3fd19dcb99..0000000000
--- a/be/src/util/storage_backend_mgr.h
+++ /dev/null
@@ -1,77 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <map>
-#include <memory>
-#include <shared_mutex>
-#include <string>
-#include <vector>
-
-#include "common/status.h"
-
-namespace doris {
-
-class StorageBackend;
-class StorageParamPB;
-
-// StorageBackendMgr is used to manage StorageBackend, it has (key -> StorageBackend) map used to connect remote storage
-class StorageBackendMgr {
-public:
- StorageBackendMgr() {}
- ~StorageBackendMgr() {}
-
- static StorageBackendMgr* instance() {
- static StorageBackendMgr s_instance;
- return &s_instance;
- }
-
- // init() is called when be is started, storage_name_dir is the file path for remote parameter in local cache_path.
- Status init(const std::string& storage_name_dir);
-
- // get_storage_backend by storage_name, one storage_name matches a remote storage_backend.
- std::shared_ptr<StorageBackend> get_storage_backend(const std::string& storage_name);
-
- // create a new remote storage_backend when it doesn't exist.
- Status create_remote_storage(const StorageParamPB& storage_param);
-
- // get storage_param by storage_name.
- Status get_storage_param(const std::string& storage_name, StorageParamPB* storage_param);
-
- // get root_path of remote storage by storage_name
- Status get_root_path(const std::string& storage_name, std::string* root_path);
-
- // get root_path of remote storage from storage_param
- static std::string get_root_path_from_param(const StorageParamPB& storage_param);
-
-private:
- Status _create_remote_storage_internal(const StorageParamPB& storage_param);
- Status _check_exist(const StorageParamPB& storage_param_pb);
- Status _serialize_param(const StorageParamPB& storage_param_pb, std::string* meta_binary);
- Status _deserialize_param(const std::string& meta_binary, StorageParamPB* storage_param_pb);
-
- std::shared_mutex _storage_backend_lock;
- std::map<std::string, time_t> _storage_backend_active_time;
- // key is storage_name, value is StorageBackend with one storage_backend.
- std::map<std::string, std::shared_ptr<StorageBackend>> _storage_backend_map;
- std::map<std::string, StorageParamPB> _storage_param_map;
- std::string _storage_param_dir;
- bool _is_inited = false;
-};
-
-} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/core/block_spill_reader.cpp b/be/src/vec/core/block_spill_reader.cpp
index 945d14f9ff..c7871f30a5 100644
--- a/be/src/vec/core/block_spill_reader.cpp
+++ b/be/src/vec/core/block_spill_reader.cpp
@@ -38,9 +38,8 @@ Status BlockSpillReader::open() {
FileDescription file_description;
file_description.path = file_path_;
- IOContext io_ctx;
RETURN_IF_ERROR(FileFactory::create_file_reader(nullptr, system_properties, file_description,
- &file_system, &file_reader_, &io_ctx));
+ &file_system, &file_reader_));
if (delete_after_read_) {
unlink(file_path_.c_str());
}
@@ -51,11 +50,11 @@ Status BlockSpillReader::open() {
// read block count
size_t bytes_read = 0;
- RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t), result, {}, &bytes_read));
+ RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t), result, &bytes_read));
// read max sub block size
result.data = (char*)&max_sub_block_size_;
- RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t) * 2, result, {}, &bytes_read));
+ RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t) * 2, result, &bytes_read));
size_t buff_size = std::max(block_count_ * sizeof(size_t), max_sub_block_size_);
read_buff_.reset(new char[buff_size]);
@@ -65,7 +64,7 @@ Status BlockSpillReader::open() {
result.data = read_buff_.get();
result.size = block_count_ * sizeof(size_t);
- RETURN_IF_ERROR(file_reader_->read_at(read_offset, result, {}, &bytes_read));
+ RETURN_IF_ERROR(file_reader_->read_at(read_offset, result, &bytes_read));
DCHECK(bytes_read == block_count_ * sizeof(size_t));
block_start_offsets_.resize(block_count_ + 1);
@@ -96,7 +95,7 @@ Status BlockSpillReader::read(Block* block, bool* eos) {
{
SCOPED_TIMER(read_time_);
- RETURN_IF_ERROR(file_reader_->read_at(block_start_offsets_[read_block_index_], result, {},
+ RETURN_IF_ERROR(file_reader_->read_at(block_start_offsets_[read_block_index_], result,
&bytes_read));
}
DCHECK(bytes_read == bytes_to_read);
diff --git a/be/src/vec/core/block_spill_writer.cpp b/be/src/vec/core/block_spill_writer.cpp
index c1a3d8911c..b171eac19a 100644
--- a/be/src/vec/core/block_spill_writer.cpp
+++ b/be/src/vec/core/block_spill_writer.cpp
@@ -33,9 +33,6 @@ void BlockSpillWriter::_init_profile() {
Status BlockSpillWriter::open() {
RETURN_IF_ERROR(FileFactory::create_file_writer(TFileType::FILE_LOCAL, ExecEnv::GetInstance(),
{}, {}, file_path_, 0, file_writer_));
-
- RETURN_IF_ERROR(file_writer_->open());
-
is_open_ = true;
return Status::OK();
}
@@ -53,17 +50,17 @@ Status BlockSpillWriter::close() {
meta_.append((const char*)&written_blocks_, sizeof(written_blocks_));
Status status;
- size_t written_bytes;
// meta: block1 offset, block2 offset, ..., blockn offset, n
{
SCOPED_TIMER(write_timer_);
- status = file_writer_->write((const uint8_t*)meta_.data(), meta_.size(), &written_bytes);
+ status = file_writer_->append(meta_);
}
if (!status.ok()) {
unlink(file_path_.c_str());
return status;
}
+ RETURN_IF_ERROR(file_writer_->close());
file_writer_.reset();
return Status::OK();
}
@@ -125,13 +122,13 @@ Status BlockSpillWriter::_write_internal(const Block& block) {
{
SCOPED_TIMER(write_timer_);
- status = file_writer_->write((const uint8_t*)buff.data(), buff.size(), &written_bytes);
+ status = file_writer_->append(buff);
+ written_bytes = buff.size();
}
if (!status.ok()) {
unlink(file_path_.c_str());
return status;
}
- DCHECK(written_bytes == buff.size());
max_sub_block_size_ = std::max(max_sub_block_size_, written_bytes);
@@ -144,4 +141,4 @@ Status BlockSpillWriter::_write_internal(const Block& block) {
}
} // namespace vectorized
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/vec/core/block_spill_writer.h b/be/src/vec/core/block_spill_writer.h
... 3070 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org