You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pegasus.apache.org by GitBox <gi...@apache.org> on 2022/08/10 07:34:51 UTC
[GitHub] [incubator-pegasus] hycdong opened a new pull request, #1109: feat: add some functions for block service manager
hycdong opened a new pull request, #1109:
URL: https://github.com/apache/incubator-pegasus/pull/1109
https://github.com/apache/incubator-pegasus/issues/1081
This pr adds following functions for block service manager:
- upload_file
- read_file
- write_file
- remove_path
All of them will be used for backup and restore in further prs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org
[GitHub] [incubator-pegasus] acelyc111 merged pull request #1109: feat: add some functions for block service manager
Posted by GitBox <gi...@apache.org>.
acelyc111 merged PR #1109:
URL: https://github.com/apache/incubator-pegasus/pull/1109
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org
[GitHub] [incubator-pegasus] foreverneverer commented on a diff in pull request #1109: feat: add some functions for block service manager
Posted by GitBox <gi...@apache.org>.
foreverneverer commented on code in PR #1109:
URL: https://github.com/apache/incubator-pegasus/pull/1109#discussion_r943151597
##########
src/rdsn/src/block_service/block_service_manager.cpp:
##########
@@ -184,6 +184,140 @@ error_code block_service_manager::download_file(const std::string &remote_dir,
return ERR_OK;
}
+static upload_response
+upload_block_file_sync(const std::string &local_file_path, block_file *bf, task_tracker *tracker)
+{
+ upload_response ret;
+ bf->upload(upload_request{local_file_path},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const upload_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::upload_file(const std::string &remote_dir,
+ const std::string &local_dir,
+ const std::string &file_name,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
+ block_file_ptr bf = create_resp.file_handle;
+
+ // Upload file
+ const auto &local_file_name = utils::filesystem::path_combine(local_dir, file_name);
+ const upload_response &resp = upload_block_file_sync(local_file_name, bf.get(), &tracker);
+ if (resp.err != ERR_OK) {
+ return resp.err;
+ }
+ ddebug_f("upload file({}) succeed", local_file_name);
+ return ERR_OK;
+}
+
+static write_response
+write_block_file_sync(const blob &value, block_file *bf, task_tracker *tracker)
+{
+ write_response ret;
+ bf->write(write_request{value},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const write_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::write_file(const std::string &remote_dir,
+ const std::string &file_name,
+ const blob &value,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
Review Comment:
same
##########
src/rdsn/src/block_service/block_service_manager.cpp:
##########
@@ -184,6 +184,140 @@ error_code block_service_manager::download_file(const std::string &remote_dir,
return ERR_OK;
}
+static upload_response
+upload_block_file_sync(const std::string &local_file_path, block_file *bf, task_tracker *tracker)
+{
+ upload_response ret;
+ bf->upload(upload_request{local_file_path},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const upload_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::upload_file(const std::string &remote_dir,
+ const std::string &local_dir,
+ const std::string &file_name,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
Review Comment:
delete
##########
src/rdsn/src/block_service/block_service_manager.cpp:
##########
@@ -184,6 +184,140 @@ error_code block_service_manager::download_file(const std::string &remote_dir,
return ERR_OK;
}
+static upload_response
+upload_block_file_sync(const std::string &local_file_path, block_file *bf, task_tracker *tracker)
+{
+ upload_response ret;
+ bf->upload(upload_request{local_file_path},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const upload_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::upload_file(const std::string &remote_dir,
+ const std::string &local_dir,
+ const std::string &file_name,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
+ block_file_ptr bf = create_resp.file_handle;
+
+ // Upload file
+ const auto &local_file_name = utils::filesystem::path_combine(local_dir, file_name);
+ const upload_response &resp = upload_block_file_sync(local_file_name, bf.get(), &tracker);
+ if (resp.err != ERR_OK) {
+ return resp.err;
+ }
+ ddebug_f("upload file({}) succeed", local_file_name);
+ return ERR_OK;
+}
+
+static write_response
+write_block_file_sync(const blob &value, block_file *bf, task_tracker *tracker)
+{
+ write_response ret;
+ bf->write(write_request{value},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const write_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::write_file(const std::string &remote_dir,
+ const std::string &file_name,
+ const blob &value,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
+ block_file_ptr bf = create_resp.file_handle;
+
+ // Write blob
+ const write_response &resp = write_block_file_sync(value, bf.get(), &tracker);
+ if (resp.err != ERR_OK) {
+ return resp.err;
+ }
+ ddebug_f("write remote file({}) succeed", remote_file_name);
+ return ERR_OK;
+}
+
+error_code
+block_service_manager::remove_path(const std::string &path, bool recursive, block_filesystem *fs)
+{
+ task_tracker tracker;
+ remove_path_response ret;
+ fs->remove_path(remove_path_request{path, recursive},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const remove_path_response &resp) { ret = resp; },
+ &tracker);
+ tracker.wait_outstanding_tasks();
+ return ret.err;
+}
+
+static read_response read_block_file_sync(block_file *bf,
+ const uint64_t remote_pos,
+ const int64_t remote_length,
+ task_tracker *tracker)
+{
+ read_response ret;
+ bf->read(read_request{remote_pos, remote_length},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const read_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::read_file(const std::string &remote_dir,
+ const std::string &file_name,
+ block_filesystem *fs,
+ blob &value)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
Review Comment:
the `create` logic seem to be same, refactor it one function?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org
[GitHub] [incubator-pegasus] hycdong commented on a diff in pull request #1109: feat: add some functions for block service manager
Posted by GitBox <gi...@apache.org>.
hycdong commented on code in PR #1109:
URL: https://github.com/apache/incubator-pegasus/pull/1109#discussion_r943194929
##########
src/rdsn/src/block_service/block_service_manager.cpp:
##########
@@ -184,6 +184,140 @@ error_code block_service_manager::download_file(const std::string &remote_dir,
return ERR_OK;
}
+static upload_response
+upload_block_file_sync(const std::string &local_file_path, block_file *bf, task_tracker *tracker)
+{
+ upload_response ret;
+ bf->upload(upload_request{local_file_path},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const upload_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::upload_file(const std::string &remote_dir,
+ const std::string &local_dir,
+ const std::string &file_name,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
+ block_file_ptr bf = create_resp.file_handle;
+
+ // Upload file
+ const auto &local_file_name = utils::filesystem::path_combine(local_dir, file_name);
+ const upload_response &resp = upload_block_file_sync(local_file_name, bf.get(), &tracker);
+ if (resp.err != ERR_OK) {
+ return resp.err;
+ }
+ ddebug_f("upload file({}) succeed", local_file_name);
+ return ERR_OK;
+}
+
+static write_response
+write_block_file_sync(const blob &value, block_file *bf, task_tracker *tracker)
+{
+ write_response ret;
+ bf->write(write_request{value},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const write_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::write_file(const std::string &remote_dir,
+ const std::string &file_name,
+ const blob &value,
+ block_filesystem *fs)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
+ block_file_ptr bf = create_resp.file_handle;
+
+ // Write blob
+ const write_response &resp = write_block_file_sync(value, bf.get(), &tracker);
+ if (resp.err != ERR_OK) {
+ return resp.err;
+ }
+ ddebug_f("write remote file({}) succeed", remote_file_name);
+ return ERR_OK;
+}
+
+error_code
+block_service_manager::remove_path(const std::string &path, bool recursive, block_filesystem *fs)
+{
+ task_tracker tracker;
+ remove_path_response ret;
+ fs->remove_path(remove_path_request{path, recursive},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const remove_path_response &resp) { ret = resp; },
+ &tracker);
+ tracker.wait_outstanding_tasks();
+ return ret.err;
+}
+
+static read_response read_block_file_sync(block_file *bf,
+ const uint64_t remote_pos,
+ const int64_t remote_length,
+ task_tracker *tracker)
+{
+ read_response ret;
+ bf->read(read_request{remote_pos, remote_length},
+ TASK_CODE_EXEC_INLINED,
+ [&ret](const read_response &resp) { ret = resp; },
+ tracker);
+ tracker->wait_outstanding_tasks();
+ return ret;
+}
+
+error_code block_service_manager::read_file(const std::string &remote_dir,
+ const std::string &file_name,
+ block_filesystem *fs,
+ blob &value)
+{
+ task_tracker tracker;
+
+ // Create a block_file object.
+ const auto &remote_file_name = utils::filesystem::path_combine(remote_dir, file_name);
+ const auto &create_resp =
+ create_block_file_sync(remote_file_name, false /*ignore file meta*/, fs, &tracker);
+ const auto &err = create_resp.err;
+ if (err != ERR_OK) {
+ derror_f("create file({}) failed with error({})", remote_file_name, err.to_string());
+ return err;
+ }
Review Comment:
Good idea, Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org
[GitHub] [incubator-pegasus] GiantKing commented on pull request #1109: feat: add some functions for block service manager
Posted by GitBox <gi...@apache.org>.
GiantKing commented on PR #1109:
URL: https://github.com/apache/incubator-pegasus/pull/1109#issuecomment-1212674223
LGTM
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org