You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2020/01/16 22:46:52 UTC
[kudu] 02/03: tools: avoid extra 100ms sleep at end of table scan
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 6325c4edcc389aa2ee1d8f028f367978047b8be6
Author: Todd Lipcon <to...@apache.org>
AuthorDate: Wed Jan 15 15:23:00 2020 -0800
tools: avoid extra 100ms sleep at end of table scan
Prior to this patch, the table scan tool printed a status output every
5 seconds by starting a separate monitor thread. This thread's main loop
would only check for the scan completion every 100ms, so after the scans
completed, the process could hang for up to an extra 100ms before
exiting.
This changes the printing to happen from the main thread instead, with
the waiting based on the actual scan completion. Now the tool exits
immediately when the scan completes.
Example output:
Before:
todd@turbo:~/kudu$ time ./build/latest/bin/kudu table scan localhost sequences | wc -l
79
real 0m0.180s
user 0m0.041s
sys 0m0.021s
After:
todd@turbo:~/kudu$ time ./build/latest/bin/kudu table scan localhost sequences | wc -l
79
real 0m0.078s
user 0m0.055s
sys 0m0.005s
Change-Id: Ic7dcd8408b1dbd5322a2f5e8f4889f5f5660ece0
Reviewed-on: http://gerrit.cloudera.org:8080/15040
Reviewed-by: Bankim Bhavsar <ba...@cloudera.com>
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Grant Henke <gr...@apache.org>
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
src/kudu/tools/table_scanner.cc | 20 ++++++--------------
src/kudu/tools/table_scanner.h | 1 -
2 files changed, 6 insertions(+), 15 deletions(-)
diff --git a/src/kudu/tools/table_scanner.cc b/src/kudu/tools/table_scanner.cc
index 4924f30..f50b93a 100644
--- a/src/kudu/tools/table_scanner.cc
+++ b/src/kudu/tools/table_scanner.cc
@@ -502,8 +502,9 @@ void TableScanner::ScanTask(const vector<KuduScanToken *>& tokens, Status* threa
if (out_ && FLAGS_show_values) {
MutexLock l(output_lock_);
for (const auto& row : batch) {
- *out_ << row.ToString() << endl;
+ *out_ << row.ToString() << "\n";
}
+ out_->flush();
}
});
}
@@ -530,16 +531,6 @@ void TableScanner::CopyTask(const vector<KuduScanToken*>& tokens, Status* thread
});
}
-void TableScanner::MonitorTask() {
- MonoTime last_log_time = MonoTime::Now();
- while (thread_pool_->num_threads() > 1) { // Some other table scan thread is running.
- if (MonoTime::Now() - last_log_time >= MonoDelta::FromSeconds(5)) {
- LOG(INFO) << "Scanned count: " << total_count_.Load();
- last_log_time = MonoTime::Now();
- }
- SleepFor(MonoDelta::FromMilliseconds(100));
- }
-}
void TableScanner::SetOutput(ostream* out) {
out_ = out;
@@ -600,7 +591,7 @@ Status TableScanner::StartWork(WorkType type) {
vector<Status> thread_statuses(FLAGS_num_threads);
RETURN_NOT_OK(ThreadPoolBuilder("table_scan_pool")
- .set_max_threads(FLAGS_num_threads + 1) // add extra 1 thread for MonitorTask
+ .set_max_threads(FLAGS_num_threads)
.set_idle_timeout(MonoDelta::FromMilliseconds(1))
.Build(&thread_pool_));
@@ -617,8 +608,9 @@ Status TableScanner::StartWork(WorkType type) {
boost::bind(&TableScanner::CopyTask, this, thread_tokens[i], &thread_statuses[i])));
}
}
- RETURN_NOT_OK(thread_pool_->SubmitFunc(boost::bind(&TableScanner::MonitorTask, this)));
- thread_pool_->Wait();
+ while (!thread_pool_->WaitFor(MonoDelta::FromSeconds(5))) {
+ LOG(INFO) << "Scanned count: " << total_count_.Load();
+ }
thread_pool_->Shutdown();
sw.stop();
diff --git a/src/kudu/tools/table_scanner.h b/src/kudu/tools/table_scanner.h
index b784835..ff6b81b 100644
--- a/src/kudu/tools/table_scanner.h
+++ b/src/kudu/tools/table_scanner.h
@@ -83,7 +83,6 @@ class TableScanner {
const std::function<void(const kudu::client::KuduScanBatch& batch)>& cb);
void ScanTask(const std::vector<kudu::client::KuduScanToken*>& tokens, Status* thread_status);
void CopyTask(const std::vector<kudu::client::KuduScanToken*>& tokens, Status* thread_status);
- void MonitorTask();
Status AddRow(const client::sp::shared_ptr<kudu::client::KuduTable>& table,
const kudu::client::KuduSchema& table_schema,