You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by yu...@apache.org on 2022/01/21 08:31:19 UTC
[incubator-pegasus] branch master updated: feat(bulk_load): support verify_before_ingest option (#888)
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 0ff2ce6 feat(bulk_load): support verify_before_ingest option (#888)
0ff2ce6 is described below
commit 0ff2ce6b38aa59a0154e2cac7c5922cac531b69a
Author: HeYuchen <he...@xiaomi.com>
AuthorDate: Fri Jan 21 16:28:55 2022 +0800
feat(bulk_load): support verify_before_ingest option (#888)
---
rdsn | 2 +-
src/server/pegasus_write_service_impl.h | 13 +++++++------
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/rdsn b/rdsn
index 0cb88e2..34a3b26 160000
--- a/rdsn
+++ b/rdsn
@@ -1 +1 @@
-Subproject commit 0cb88e20076936da62bc245451d26fc354cbd4b2
+Subproject commit 34a3b26d895304445d1c14ce52e47dcc39cad172
diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h
index 2f79953..1a8884c 100644
--- a/src/server/pegasus_write_service_impl.h
+++ b/src/server/pegasus_write_service_impl.h
@@ -67,17 +67,17 @@ inline int get_cluster_id_if_exists()
}
inline dsn::error_code get_external_files_path(const std::string &bulk_load_dir,
+ const bool verify_before_ingest,
const dsn::replication::bulk_load_metadata &metadata,
/*out*/ std::vector<std::string> &files_path)
{
for (const auto &f_meta : metadata.files) {
- const std::string &file_name =
- dsn::utils::filesystem::path_combine(bulk_load_dir, f_meta.name);
- if (dsn::utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) {
- files_path.emplace_back(file_name);
- } else {
+ const auto &file_name = dsn::utils::filesystem::path_combine(bulk_load_dir, f_meta.name);
+ if (verify_before_ingest &&
+ !dsn::utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) {
break;
}
+ files_path.emplace_back(file_name);
}
return files_path.size() == metadata.files.size() ? dsn::ERR_OK : dsn::ERR_WRONG_CHECKSUM;
}
@@ -502,7 +502,8 @@ public:
// verify external files before ingestion
std::vector<std::string> sst_file_list;
- const auto &err = get_external_files_path(bulk_load_dir, req.metadata, sst_file_list);
+ const auto &err = get_external_files_path(
+ bulk_load_dir, req.verify_before_ingest, req.metadata, sst_file_list);
if (err != dsn::ERR_OK) {
return err;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org