You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by yu...@apache.org on 2020/12/29 03:32:54 UTC
[incubator-pegasus] 04/07: feat(bulk_load): add remote_file_root
for start_bulk_load_request (#660)
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch v2.1.2
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 03bcd8894ba3dc7b0307db98c69ae41f5d70dc02
Author: HeYuchen <37...@qq.com>
AuthorDate: Mon Dec 21 19:24:07 2020 +0800
feat(bulk_load): add remote_file_root for start_bulk_load_request (#660)
---
src/server/config.ini | 1 -
src/server/config.min.ini | 1 -
src/shell/commands/bulk_load.cpp | 15 +++++++++++++--
src/shell/main.cpp | 3 ++-
src/test/function_test/test_bulk_load.cpp | 2 +-
5 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/src/server/config.ini b/src/server/config.ini
index 5dbda09..2e39196 100644
--- a/src/server/config.ini
+++ b/src/server/config.ini
@@ -266,7 +266,6 @@
cold_backup_root = %{cluster.name}
max_concurrent_uploading_file_count = 10
- bulk_load_provider_root = bulk_load_root
max_concurrent_bulk_load_downloading_count = 5
[pegasus.server]
diff --git a/src/server/config.min.ini b/src/server/config.min.ini
index a12fe8d..0094879 100644
--- a/src/server/config.min.ini
+++ b/src/server/config.min.ini
@@ -117,7 +117,6 @@
duplication_disabled = true
cluster_name = onebox
cold_backup_checkpoint_reserve_minutes = 10
- bulk_load_provider_root = bulk_load_root
[meta_server.apps.@APP_NAME@]
app_name = @APP_NAME@
diff --git a/src/shell/commands/bulk_load.cpp b/src/shell/commands/bulk_load.cpp
index 19a0c35..dd8fc93 100644
--- a/src/shell/commands/bulk_load.cpp
+++ b/src/shell/commands/bulk_load.cpp
@@ -24,16 +24,18 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
static struct option long_options[] = {{"app_name", required_argument, 0, 'a'},
{"cluster_name", required_argument, 0, 'c'},
{"file_provider_type", required_argument, 0, 'p'},
+ {"root_path", required_argument, 0, 'r'},
{0, 0, 0, 0}};
std::string app_name;
std::string cluster_name;
std::string file_provider_type;
+ std::string remote_root_path;
optind = 0;
while (true) {
int option_index = 0;
int c;
- c = getopt_long(args.argc, args.argv, "a:c:p:", long_options, &option_index);
+ c = getopt_long(args.argc, args.argv, "a:c:p:r:", long_options, &option_index);
if (c == -1)
break;
switch (c) {
@@ -46,6 +48,9 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
case 'p':
file_provider_type = optarg;
break;
+ case 'r':
+ remote_root_path = optarg;
+ break;
default:
return false;
}
@@ -65,7 +70,13 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
return false;
}
- auto err_resp = sc->ddl_client->start_bulk_load(app_name, cluster_name, file_provider_type);
+ if (remote_root_path.empty()) {
+ fprintf(stderr, "remote_root_path should not be empty\n");
+ return false;
+ }
+
+ auto err_resp = sc->ddl_client->start_bulk_load(
+ app_name, cluster_name, file_provider_type, remote_root_path);
dsn::error_s err = err_resp.get_error();
std::string hint_msg;
if (err.is_ok()) {
diff --git a/src/shell/main.cpp b/src/shell/main.cpp
index 050a40f..90601d3 100644
--- a/src/shell/main.cpp
+++ b/src/shell/main.cpp
@@ -468,7 +468,8 @@ static command_executor commands[] = {
{
"start_bulk_load",
"start app bulk load",
- "<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str>",
+ "<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str> <-r "
+ "--root_path>",
start_bulk_load,
},
{
diff --git a/src/test/function_test/test_bulk_load.cpp b/src/test/function_test/test_bulk_load.cpp
index 963d419..7f5c11b 100644
--- a/src/test/function_test/test_bulk_load.cpp
+++ b/src/test/function_test/test_bulk_load.cpp
@@ -103,7 +103,7 @@ public:
error_code start_bulk_load()
{
- auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER);
+ auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER, LOCAL_ROOT);
return err_resp.get_value().err;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org