You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/09/01 21:58:16 UTC
[2/2] kudu git commit: transaction_tracker: back-off when logging
in-flight transactions
transaction_tracker: back-off when logging in-flight transactions
On a test cluster pushed into overload, I ended up with thousands of
in-flight transactions which were only being drained at the rate of one
or two per second. Dumping the entire list of transactions once a second
ended up flooding the glog with many MB per second work of transaction
strings.
This changes the dumping to back-off exponentially.
Change-Id: I7d1a2563dd57c59ae8c130a5b26966f4964048de
Reviewed-on: http://gerrit.cloudera.org:8080/4159
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
Reviewed-by: David Ribeiro Alves <dr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/075c42cb
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/075c42cb
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/075c42cb
Branch: refs/heads/master
Commit: 075c42cb78a80042040850eb5746c28e663a4622
Parents: 8a4bb33
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Aug 29 12:52:01 2016 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Sep 1 21:57:51 2016 +0000
----------------------------------------------------------------------
.../tablet/transactions/transaction_tracker.cc | 29 +++++++++++---------
1 file changed, 16 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/075c42cb/src/kudu/tablet/transactions/transaction_tracker.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/transactions/transaction_tracker.cc b/src/kudu/tablet/transactions/transaction_tracker.cc
index fbdb5a6..f2bc6c4 100644
--- a/src/kudu/tablet/transactions/transaction_tracker.cc
+++ b/src/kudu/tablet/transactions/transaction_tracker.cc
@@ -201,10 +201,11 @@ void TransactionTracker::WaitForAllToFinish() const {
}
Status TransactionTracker::WaitForAllToFinish(const MonoDelta& timeout) const {
- const int complain_ms = 1000;
- int wait_time = 250;
+ int wait_time_us = 250;
int num_complaints = 0;
MonoTime start_time = MonoTime::Now();
+ MonoTime next_log_time = start_time + MonoDelta::FromSeconds(1);
+
while (1) {
vector<scoped_refptr<TransactionDriver> > txns;
GetPendingTransactions(&txns);
@@ -213,25 +214,27 @@ Status TransactionTracker::WaitForAllToFinish(const MonoDelta& timeout) const {
break;
}
- MonoDelta diff = MonoTime::Now() - start_time;
+ MonoTime now = MonoTime::Now();
+ MonoDelta diff = now - start_time;
if (diff > timeout) {
return Status::TimedOut(Substitute("Timed out waiting for all transactions to finish. "
"$0 transactions pending. Waited for $1",
txns.size(), diff.ToString()));
}
- int64_t waited_ms = diff.ToMilliseconds();
- if (waited_ms / complain_ms > num_complaints) {
+ if (now > next_log_time) {
LOG(WARNING) << Substitute("TransactionTracker waiting for $0 outstanding transactions to"
- " complete now for $1 ms", txns.size(), waited_ms);
- num_complaints++;
- }
- wait_time = std::min(wait_time * 5 / 4, 1000000);
+ " complete now for $1", txns.size(), diff.ToString());
+ LOG(INFO) << "Dumping currently running transactions: ";
+ for (scoped_refptr<TransactionDriver> driver : txns) {
+ LOG(INFO) << driver->ToString();
+ }
- LOG(INFO) << "Dumping currently running transactions: ";
- for (scoped_refptr<TransactionDriver> driver : txns) {
- LOG(INFO) << driver->ToString();
+ num_complaints++;
+ // Exponential back-off on how often the transactions are dumped.
+ next_log_time = now + MonoDelta::FromSeconds(1 << std::min(8, num_complaints));
}
- SleepFor(MonoDelta::FromMicroseconds(wait_time));
+ wait_time_us = std::min(wait_time_us * 5 / 4, 1000000);
+ SleepFor(MonoDelta::FromMicroseconds(wait_time_us));
}
return Status::OK();
}