You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2021/11/22 10:33:14 UTC
[GitHub] [incubator-doris] morningman commented on a change in pull request #7098: Support remote storage, step1: use a struct instead of string for parameter path, add basic remote method
morningman commented on a change in pull request #7098:
URL: https://github.com/apache/incubator-doris/pull/7098#discussion_r754038577
##########
File path: be/src/env/env_remote.cpp
##########
@@ -0,0 +1,333 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
Review comment:
header is incorrect
##########
File path: be/src/env/env_remote.cpp
##########
@@ -0,0 +1,333 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors
+
+#include "common/logging.h"
+#include "common/status.h"
+#include "env/env.h"
+#include "env/env_remote.h"
+#include "util/s3_storage_backend.h"
+#include "util/s3_util.h"
+
+#include "gutil/strings/substitute.h"
+
+namespace doris {
+
+using std::string;
+using strings::Substitute;
+
+std::map<std::string, std::string> RemoteEnv::_storage_prop;
+
+class RemoteRandomAccessFile : public RandomAccessFile {
+public:
+ RemoteRandomAccessFile(std::string filename, std::map<std::string, std::string> storage_prop)
+ : _filename(std::move(filename)), _storage_backend(new S3StorageBackend(storage_prop)) {}
+ ~RemoteRandomAccessFile() {
+ }
+
+ Status read_at(uint64_t offset, const Slice* result) const override {
+ return readv_at(offset, result, 1);
+ }
+
+ Status readv_at(uint64_t offset, const Slice* result, size_t res_cnt) const override {
+ return Status::IOError("No support", 1, "");
Review comment:
```suggestion
return Status::NotSupport("No support", 1, "");
```
##########
File path: be/src/olap/data_dir.cpp
##########
@@ -77,9 +74,13 @@ DataDir::DataDir(const std::string& path, int64_t capacity_bytes,
_tablet_manager(tablet_manager),
_txn_manager(txn_manager),
_cluster_id(-1),
+ _cluster_id_incomplete(false),
_to_be_deleted(false),
_current_shard(0),
_meta(nullptr) {
+ _env = Env::get_env(storage_medium);
+ _path_desc.storage_medium = storage_medium;
Review comment:
put these to initialize list
##########
File path: be/src/olap/fs/block_manager.h
##########
@@ -229,7 +236,7 @@ class BlockManager {
// may fail.
//
// Does not modify 'block' on error.
- virtual Status open_block(const std::string& path, std::unique_ptr<ReadableBlock>* block) = 0;
+ virtual Status open_block(FilePathDesc path_desc, std::unique_ptr<ReadableBlock>* block) = 0;
Review comment:
```suggestion
virtual Status open_block(const FilePathDesc& path_desc, std::unique_ptr<ReadableBlock>* block) = 0;
```
##########
File path: be/src/olap/fs/file_block_manager.cpp
##########
@@ -63,7 +63,7 @@ namespace internal {
// FileWritableBlock instances is expected to be low.
class FileWritableBlock : public WritableBlock {
public:
- FileWritableBlock(FileBlockManager* block_manager, string path,
+ FileWritableBlock(FileBlockManager* block_manager, FilePathDesc path_desc,
Review comment:
```suggestion
FileWritableBlock(FileBlockManager* block_manager, const FilePathDesc& path_desc,
```
##########
File path: be/src/util/file_utils.cpp
##########
@@ -41,58 +41,20 @@ namespace doris {
using strings::Substitute;
Status FileUtils::create_dir(const std::string& path, Env* env) {
- if (path.empty()) {
- return Status::InvalidArgument(strings::Substitute("Unknown primitive type($0)", path));
- }
-
- std::filesystem::path p(path);
-
- std::string partial_path;
- for (std::filesystem::path::iterator it = p.begin(); it != p.end(); ++it) {
- partial_path = partial_path + it->string() + "/";
- bool is_dir = false;
-
- Status s = env->is_directory(partial_path, &is_dir);
-
- if (s.ok()) {
- if (is_dir) {
- // It's a normal directory.
- continue;
- }
-
- // Maybe a file or a symlink. Let's try to follow the symlink.
- std::string real_partial_path;
- RETURN_IF_ERROR(env->canonicalize(partial_path, &real_partial_path));
-
- RETURN_IF_ERROR(env->is_directory(real_partial_path, &is_dir));
- if (is_dir) {
- // It's a symlink to a directory.
- continue;
- } else {
- return Status::IOError(partial_path + " exists but is not a directory");
- }
- }
-
- RETURN_IF_ERROR(env->create_dir_if_missing(partial_path));
- }
-
- return Status::OK();
+ return env->create_dirs(path);
}
Status FileUtils::create_dir(const std::string& dir_path) {
return create_dir(dir_path, Env::Default());
}
Status FileUtils::remove_all(const std::string& file_path) {
- std::filesystem::path boost_path(file_path);
- std::error_code ec;
- std::filesystem::remove_all(boost_path, ec);
- if (ec) {
- std::stringstream ss;
- ss << "remove all(" << file_path << ") failed, because: " << ec;
- return Status::InternalError(ss.str());
- }
- return Status::OK();
+ return remove_all(file_path, TStorageMedium::SSD);
Review comment:
why default is SSD
##########
File path: be/src/olap/data_dir.h
##########
@@ -130,17 +136,19 @@ class DataDir {
void disks_compaction_num_increment(int64_t delta);
+ Env* env() {
+ return _env;
+ }
+
private:
- std::string _cluster_id_path() const { return _path + CLUSTER_ID_PREFIX; }
Status _init_cluster_id();
Status _init_capacity();
- Status _init_file_system();
Status _init_meta();
Status _check_disk();
OLAPStatus _read_and_write_test_file();
- Status _read_cluster_id(const std::string& cluster_id_path, int32_t* cluster_id);
- Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id);
+ Status read_cluster_id(Env* env, const std::string& cluster_id_path, int32_t* cluster_id);
Review comment:
```suggestion
Status _read_cluster_id(Env* env, const std::string& cluster_id_path, int32_t* cluster_id);
```
##########
File path: be/src/olap/fs/file_block_manager.cpp
##########
@@ -117,10 +117,10 @@ class FileWritableBlock : public WritableBlock {
size_t _bytes_appended;
};
-FileWritableBlock::FileWritableBlock(FileBlockManager* block_manager, string path,
+FileWritableBlock::FileWritableBlock(FileBlockManager* block_manager, FilePathDesc path_desc,
Review comment:
Better use `const FilePathDesc& path_desc` in parameter and remove `std::move`.
Seems meaningless to use `std::move()` here.
##########
File path: be/src/olap/rowset/segment_v2/segment.cpp
##########
@@ -37,16 +37,18 @@ namespace segment_v2 {
using strings::Substitute;
-Status Segment::open(std::string filename, uint32_t segment_id, const TabletSchema* tablet_schema,
+Status Segment::open(FilePathDesc path_desc, uint32_t segment_id, const TabletSchema* tablet_schema,
std::shared_ptr<Segment>* output) {
- std::shared_ptr<Segment> segment(new Segment(std::move(filename), segment_id, tablet_schema));
- RETURN_IF_ERROR(segment->_open());
+ std::shared_ptr<Segment> segment(new Segment(std::move(path_desc), segment_id, tablet_schema));
+ if (!Env::get_env(path_desc.storage_medium)->is_remote_env()) {
Review comment:
path_desc has been `std::move`, can use it again.
##########
File path: be/src/env/env_remote.cpp
##########
@@ -0,0 +1,333 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors
+
+#include "common/logging.h"
+#include "common/status.h"
+#include "env/env.h"
+#include "env/env_remote.h"
+#include "util/s3_storage_backend.h"
+#include "util/s3_util.h"
+
+#include "gutil/strings/substitute.h"
+
+namespace doris {
+
+using std::string;
+using strings::Substitute;
+
+std::map<std::string, std::string> RemoteEnv::_storage_prop;
+
+class RemoteRandomAccessFile : public RandomAccessFile {
+public:
+ RemoteRandomAccessFile(std::string filename, std::map<std::string, std::string> storage_prop)
+ : _filename(std::move(filename)), _storage_backend(new S3StorageBackend(storage_prop)) {}
+ ~RemoteRandomAccessFile() {
+ }
+
+ Status read_at(uint64_t offset, const Slice* result) const override {
+ return readv_at(offset, result, 1);
+ }
+
+ Status readv_at(uint64_t offset, const Slice* result, size_t res_cnt) const override {
+ return Status::IOError("No support", 1, "");
Review comment:
Same as follow
##########
File path: be/src/env/env_remote.cpp
##########
@@ -0,0 +1,333 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors
+
+#include "common/logging.h"
+#include "common/status.h"
+#include "env/env.h"
+#include "env/env_remote.h"
+#include "util/s3_storage_backend.h"
+#include "util/s3_util.h"
+
+#include "gutil/strings/substitute.h"
+
+namespace doris {
+
+using std::string;
+using strings::Substitute;
+
+std::map<std::string, std::string> RemoteEnv::_storage_prop;
+
+class RemoteRandomAccessFile : public RandomAccessFile {
+public:
+ RemoteRandomAccessFile(std::string filename, std::map<std::string, std::string> storage_prop)
+ : _filename(std::move(filename)), _storage_backend(new S3StorageBackend(storage_prop)) {}
+ ~RemoteRandomAccessFile() {
+ }
+
+ Status read_at(uint64_t offset, const Slice* result) const override {
+ return readv_at(offset, result, 1);
+ }
+
+ Status readv_at(uint64_t offset, const Slice* result, size_t res_cnt) const override {
+ return Status::IOError("No support", 1, "");
+ }
+ Status read_all(std::string* content) const override {
+ return _storage_backend->direct_download(_filename, content);
+ }
+ Status size(uint64_t* size) const override {
+ return Status::IOError("No support", 1, "");
+ }
+
+ const std::string& file_name() const override { return _filename; }
+
+private:
+ const std::string _filename;
+ std::unique_ptr<StorageBackend> _storage_backend;
+};
+
+class RemoteWritableFile : public WritableFile {
+public:
+ RemoteWritableFile(std::string filename, std::map<std::string, std::string> storage_prop, uint64_t filesize)
+ : _filename(std::move(filename)),
+ _storage_backend(new S3StorageBackend(storage_prop)),
+ _filesize(filesize) {}
+
+ ~RemoteWritableFile() override {
+ WARN_IF_ERROR(close(), "Failed to close file, file=" + _filename);
+ }
+
+ Status append(const Slice& data) override { return appendv(&data, 1); }
+
+ Status appendv(const Slice* data, size_t data_cnt) override {
+ size_t bytes_written = 0;
+ std::string content;
+ for (size_t i = 0; i < data_cnt; i++) {
+ content += data[i].to_string();
+ bytes_written += data[i].size;
+ }
+ Status status = _storage_backend->direct_upload(_filename, content);
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "direct_upload failed: $0, err=$1", _filename, status.to_string()));
+ _filesize += bytes_written;
+ return Status::OK();
+ }
+
+ Status pre_allocate(uint64_t size) override {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status close() override {
+ return Status::OK();
+ }
+
+ Status flush(FlushMode mode) override {
+ return Status::OK();
+ }
+
+ Status sync() override {
+ return Status::OK();
+ }
+
+ uint64_t size() const override { return _filesize; }
+ const string& filename() const override { return _filename; }
+
+private:
+ std::string _filename;
+ std::unique_ptr<StorageBackend> _storage_backend;
+ uint64_t _filesize = 0;
+};
+
+class RemoteRandomRWFile : public RandomRWFile {
+public:
+ RemoteRandomRWFile(const FilePathDesc& path_desc) : _path_desc(path_desc) {}
+
+ ~RemoteRandomRWFile() { WARN_IF_ERROR(close(), "Failed to close " + _path_desc.filepath); }
+
+ virtual Status read_at(uint64_t offset, const Slice& result) const {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status readv_at(uint64_t offset, const Slice* res, size_t res_cnt) const {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status write_at(uint64_t offset, const Slice& data) {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status writev_at(uint64_t offset, const Slice* data, size_t data_cnt) {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status flush(FlushMode mode, uint64_t offset, size_t length) {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status sync() {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status close() {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status size(uint64_t* size) const {
+ return Status::IOError("No support", 1, "");
+ }
+
+ const string& filename() const { return _path_desc.filepath; }
+
+private:
+ const FilePathDesc _path_desc;
+ const bool _sync_on_close = false;
+ bool _closed = false;
+};
+
+void RemoteEnv::init_s3_conf(const std::string& ak, const std::string& sk, const std::string& endpoint,
+ const std::string& region) {
+ _storage_prop[S3_AK] = ak;
+ _storage_prop[S3_SK] = sk;
+ _storage_prop[S3_ENDPOINT] = endpoint;
+ _storage_prop[S3_REGION] = region;
+}
+
+Status RemoteEnv::new_sequential_file(const std::string& fname,
+ std::unique_ptr<SequentialFile>* result) {
+ return Status::IOError(strings::Substitute("Unable to new_sequential_file $0", fname), 0, "");
+}
+
+// get a RandomAccessFile pointer without file cache
+Status RemoteEnv::new_random_access_file(const std::string& fname,
+ std::unique_ptr<RandomAccessFile>* result) {
+ return new_random_access_file(RandomAccessFileOptions(), fname, result);
+}
+
+Status RemoteEnv::new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname,
+ std::unique_ptr<RandomAccessFile>* result) {
+ result->reset(new RemoteRandomAccessFile(fname, _storage_prop));
+ return Status::OK();
+}
+
+Status RemoteEnv::new_writable_file(const std::string& fname, std::unique_ptr<WritableFile>* result) {
+ return new_writable_file(WritableFileOptions(), fname, result);
+}
+
+Status RemoteEnv::new_writable_file(const WritableFileOptions& opts, const std::string& fname,
+ std::unique_ptr<WritableFile>* result) {
+ uint64_t file_size = 0;
+ if (opts.mode == MUST_EXIST) {
+ RETURN_IF_ERROR(get_file_size(fname, &file_size));
+ }
+ result->reset(new RemoteWritableFile(fname, _storage_prop, file_size));
+ return Status::OK();
+}
+
+Status RemoteEnv::new_random_rw_file(const std::string& fname, std::unique_ptr<RandomRWFile>* result) {
+ return new_random_rw_file(RandomRWFileOptions(), fname, result);
+}
+
+Status RemoteEnv::new_random_rw_file(const RandomRWFileOptions& opts, const std::string& fname,
+ std::unique_ptr<RandomRWFile>* result) {
+ return Status::IOError(strings::Substitute("Unable to new_random_rw_file $0", fname), 0, "");
+}
+
+Status RemoteEnv::path_exists(const std::string& fname, bool is_dir) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
+ Status status = Status::OK();
+ if (is_dir) {
+ status = storage_backend->exist_dir(fname);
+ } else {
+ status = storage_backend->exist(fname);
+ }
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "path_exists failed: $0, err=$1", fname, status.to_string()));
+ return Status::OK();
+}
+
+Status RemoteEnv::get_children(const std::string& dir, std::vector<std::string>* result) {
+ return Status::IOError(strings::Substitute("Unable to get_children $0", dir), 0, "");
+}
+
+Status RemoteEnv::iterate_dir(const std::string& dir,
+ const std::function<bool(const char*)>& cb) {
+ return Status::IOError(strings::Substitute("Unable to iterate_dir $0", dir), 0, "");
+}
+
+Status RemoteEnv::delete_file(const std::string& fname) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
+ Status status = storage_backend->rm(fname);
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "delete_file failed: $0, err=$1", fname, status.to_string()));
+ return Status::OK();
+}
+
+Status RemoteEnv::create_dir(const std::string& name) {
+ return Status::OK();
+}
+
+Status RemoteEnv::create_dir_if_missing(const string& dirname, bool* created) {
+ *created = true;
+ return Status::OK();
+}
+
+Status RemoteEnv::create_dirs(const string& dirname) {
+ return Status::OK();
+}
+
+// Delete the specified directory.
+Status RemoteEnv::delete_dir(const std::string& dirname) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
+ Status status = storage_backend->rmdir(dirname);
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "delete_dir failed: $0, err=$1", dirname, status.to_string()));
+ return Status::OK();
+}
+
+Status RemoteEnv::sync_dir(const string& dirname) {
+ return Status::IOError(strings::Substitute("Unable to sync_dir $0", dirname), 0, "");
+}
+
+Status RemoteEnv::is_directory(const std::string& path, bool* is_dir) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
+ Status status = storage_backend->exist(path);
+ if (status.ok()) {
+ *is_dir = false;
+ return Status::OK();
+ }
+ if (!status.is_not_found()) {
+ return status;
+ }
+
+ status = storage_backend->exist_dir(path);
+ if (status.ok()) {
+ *is_dir = true;
+ return Status::OK();
+ }
+ if (!status.is_not_found()) {
+ return status;
+ }
+
+ *is_dir = false;
+ return Status::OK();
+}
+
+Status RemoteEnv::canonicalize(const std::string& path, std::string* result) {
+ *result = path;
+ return Status::OK();
+}
+
+Status RemoteEnv::get_file_size(const std::string& fname, uint64_t* size) {
+ return Status::OK();
+ // return EnvBos::get_file_size(fname, size);
+}
+
+Status RemoteEnv::get_file_modified_time(const std::string& fname, uint64_t* file_mtime) {
+ return Status::IOError(strings::Substitute("Unable to get_file_modified_time $0", fname), 0, "");
+}
+
+Status RemoteEnv::copy_path(const std::string& src, const std::string& target) {
+ return Status::IOError(strings::Substitute("Unable to copy_path $0 to $1", src, target), 0, "");
+}
+
+Status RemoteEnv::rename_file(const std::string& src, const std::string& target) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
Review comment:
why we need to create a new S3StorageBackend each time we call this method?
##########
File path: be/src/env/env_remote.cpp
##########
@@ -0,0 +1,333 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors
+
+#include "common/logging.h"
+#include "common/status.h"
+#include "env/env.h"
+#include "env/env_remote.h"
+#include "util/s3_storage_backend.h"
+#include "util/s3_util.h"
+
+#include "gutil/strings/substitute.h"
+
+namespace doris {
+
+using std::string;
+using strings::Substitute;
+
+std::map<std::string, std::string> RemoteEnv::_storage_prop;
+
+class RemoteRandomAccessFile : public RandomAccessFile {
+public:
+ RemoteRandomAccessFile(std::string filename, std::map<std::string, std::string> storage_prop)
+ : _filename(std::move(filename)), _storage_backend(new S3StorageBackend(storage_prop)) {}
+ ~RemoteRandomAccessFile() {
+ }
+
+ Status read_at(uint64_t offset, const Slice* result) const override {
+ return readv_at(offset, result, 1);
+ }
+
+ Status readv_at(uint64_t offset, const Slice* result, size_t res_cnt) const override {
+ return Status::IOError("No support", 1, "");
+ }
+ Status read_all(std::string* content) const override {
+ return _storage_backend->direct_download(_filename, content);
+ }
+ Status size(uint64_t* size) const override {
+ return Status::IOError("No support", 1, "");
+ }
+
+ const std::string& file_name() const override { return _filename; }
+
+private:
+ const std::string _filename;
+ std::unique_ptr<StorageBackend> _storage_backend;
+};
+
+class RemoteWritableFile : public WritableFile {
+public:
+ RemoteWritableFile(std::string filename, std::map<std::string, std::string> storage_prop, uint64_t filesize)
+ : _filename(std::move(filename)),
+ _storage_backend(new S3StorageBackend(storage_prop)),
+ _filesize(filesize) {}
+
+ ~RemoteWritableFile() override {
+ WARN_IF_ERROR(close(), "Failed to close file, file=" + _filename);
+ }
+
+ Status append(const Slice& data) override { return appendv(&data, 1); }
+
+ Status appendv(const Slice* data, size_t data_cnt) override {
+ size_t bytes_written = 0;
+ std::string content;
+ for (size_t i = 0; i < data_cnt; i++) {
+ content += data[i].to_string();
+ bytes_written += data[i].size;
+ }
+ Status status = _storage_backend->direct_upload(_filename, content);
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "direct_upload failed: $0, err=$1", _filename, status.to_string()));
+ _filesize += bytes_written;
+ return Status::OK();
+ }
+
+ Status pre_allocate(uint64_t size) override {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status close() override {
+ return Status::OK();
+ }
+
+ Status flush(FlushMode mode) override {
+ return Status::OK();
+ }
+
+ Status sync() override {
+ return Status::OK();
+ }
+
+ uint64_t size() const override { return _filesize; }
+ const string& filename() const override { return _filename; }
+
+private:
+ std::string _filename;
+ std::unique_ptr<StorageBackend> _storage_backend;
+ uint64_t _filesize = 0;
+};
+
+class RemoteRandomRWFile : public RandomRWFile {
+public:
+ RemoteRandomRWFile(const FilePathDesc& path_desc) : _path_desc(path_desc) {}
+
+ ~RemoteRandomRWFile() { WARN_IF_ERROR(close(), "Failed to close " + _path_desc.filepath); }
+
+ virtual Status read_at(uint64_t offset, const Slice& result) const {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status readv_at(uint64_t offset, const Slice* res, size_t res_cnt) const {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status write_at(uint64_t offset, const Slice& data) {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status writev_at(uint64_t offset, const Slice* data, size_t data_cnt) {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status flush(FlushMode mode, uint64_t offset, size_t length) {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status sync() {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status close() {
+ return Status::IOError("No support", 1, "");
+ }
+
+ Status size(uint64_t* size) const {
+ return Status::IOError("No support", 1, "");
+ }
+
+ const string& filename() const { return _path_desc.filepath; }
+
+private:
+ const FilePathDesc _path_desc;
+ const bool _sync_on_close = false;
+ bool _closed = false;
+};
+
+void RemoteEnv::init_s3_conf(const std::string& ak, const std::string& sk, const std::string& endpoint,
+ const std::string& region) {
+ _storage_prop[S3_AK] = ak;
+ _storage_prop[S3_SK] = sk;
+ _storage_prop[S3_ENDPOINT] = endpoint;
+ _storage_prop[S3_REGION] = region;
+}
+
+Status RemoteEnv::new_sequential_file(const std::string& fname,
+ std::unique_ptr<SequentialFile>* result) {
+ return Status::IOError(strings::Substitute("Unable to new_sequential_file $0", fname), 0, "");
+}
+
+// get a RandomAccessFile pointer without file cache
+Status RemoteEnv::new_random_access_file(const std::string& fname,
+ std::unique_ptr<RandomAccessFile>* result) {
+ return new_random_access_file(RandomAccessFileOptions(), fname, result);
+}
+
+Status RemoteEnv::new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname,
+ std::unique_ptr<RandomAccessFile>* result) {
+ result->reset(new RemoteRandomAccessFile(fname, _storage_prop));
+ return Status::OK();
+}
+
+Status RemoteEnv::new_writable_file(const std::string& fname, std::unique_ptr<WritableFile>* result) {
+ return new_writable_file(WritableFileOptions(), fname, result);
+}
+
+Status RemoteEnv::new_writable_file(const WritableFileOptions& opts, const std::string& fname,
+ std::unique_ptr<WritableFile>* result) {
+ uint64_t file_size = 0;
+ if (opts.mode == MUST_EXIST) {
+ RETURN_IF_ERROR(get_file_size(fname, &file_size));
+ }
+ result->reset(new RemoteWritableFile(fname, _storage_prop, file_size));
+ return Status::OK();
+}
+
+Status RemoteEnv::new_random_rw_file(const std::string& fname, std::unique_ptr<RandomRWFile>* result) {
+ return new_random_rw_file(RandomRWFileOptions(), fname, result);
+}
+
+Status RemoteEnv::new_random_rw_file(const RandomRWFileOptions& opts, const std::string& fname,
+ std::unique_ptr<RandomRWFile>* result) {
+ return Status::IOError(strings::Substitute("Unable to new_random_rw_file $0", fname), 0, "");
+}
+
+Status RemoteEnv::path_exists(const std::string& fname, bool is_dir) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
+ Status status = Status::OK();
+ if (is_dir) {
+ status = storage_backend->exist_dir(fname);
+ } else {
+ status = storage_backend->exist(fname);
+ }
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "path_exists failed: $0, err=$1", fname, status.to_string()));
+ return Status::OK();
+}
+
+Status RemoteEnv::get_children(const std::string& dir, std::vector<std::string>* result) {
+ return Status::IOError(strings::Substitute("Unable to get_children $0", dir), 0, "");
+}
+
+Status RemoteEnv::iterate_dir(const std::string& dir,
+ const std::function<bool(const char*)>& cb) {
+ return Status::IOError(strings::Substitute("Unable to iterate_dir $0", dir), 0, "");
+}
+
+Status RemoteEnv::delete_file(const std::string& fname) {
+ std::unique_ptr<StorageBackend> storage_backend(new S3StorageBackend(_storage_prop));
+ Status status = storage_backend->rm(fname);
+ RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute(
+ "delete_file failed: $0, err=$1", fname, status.to_string()));
+ return Status::OK();
+}
+
+Status RemoteEnv::create_dir(const std::string& name) {
Review comment:
Add comment to explain why directly return ok.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org