You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2020/02/24 21:17:02 UTC
[kudu] 01/05: [util] move PeriodWebUIChecker into a separate header
file
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 7cfcab629c51df521aff688d111b3691a317011b
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Fri Feb 21 21:09:56 2020 -0800
[util] move PeriodWebUIChecker into a separate header file
This patch moves PeriodWebUIChecker into a separate header file.
A follow-up changelist will use its functionality in a test outside
of the integration-tests test set.
Change-Id: I2e6c8e0ebe9e5d03ddf010ea225414af5823becc
Reviewed-on: http://gerrit.cloudera.org:8080/15271
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
---
src/kudu/integration-tests/linked_list-test-util.h | 110 -----------------
src/kudu/integration-tests/linked_list-test.cc | 2 +-
.../integration-tests/webserver-stress-itest.cc | 2 +-
src/kudu/mini-cluster/CMakeLists.txt | 1 +
src/kudu/mini-cluster/webui_checker.cc | 131 +++++++++++++++++++++
src/kudu/mini-cluster/webui_checker.h | 71 +++++++++++
6 files changed, 205 insertions(+), 112 deletions(-)
diff --git a/src/kudu/integration-tests/linked_list-test-util.h b/src/kudu/integration-tests/linked_list-test-util.h
index b80088a..5547f66 100644
--- a/src/kudu/integration-tests/linked_list-test-util.h
+++ b/src/kudu/integration-tests/linked_list-test-util.h
@@ -303,116 +303,6 @@ class ScopedRowUpdater {
scoped_refptr<Thread> updater_;
};
-// A thread that periodically checks tablet and master web pages during the
-// linked list test.
-class PeriodicWebUIChecker {
- public:
- PeriodicWebUIChecker(const cluster::ExternalMiniCluster& cluster,
- MonoDelta period,
- const std::string& tablet_id = "")
- : period_(period), is_running_(true) {
- // List of master web pages to fetch.
- const std::vector<std::string> master_pages = {
- "/dump-entities",
- "/masters",
- "/mem-trackers",
- "/metrics",
- "/stacks",
- "/tables",
- "/tablet-servers",
- "/threadz",
- "/threadz?group=all",
- };
-
- // List of tserver web pages to fetch.
- const std::vector<std::string> ts_pages = {
- "/maintenance-manager",
- "/mem-trackers",
- "/metrics",
- "/scans",
- "/stacks",
- "/tablets",
- "/threadz",
- "/threadz?group=all",
- tablet_id.empty()
- ? "/transactions"
- : strings::Substitute("/transactions?tablet_id=$0", tablet_id),
- };
-
- // Generate list of urls for each master and tablet server
- for (int i = 0; i < cluster.num_masters(); i++) {
- for (const auto& page : master_pages) {
- urls_.emplace_back(strings::Substitute(
- "http://$0$1",
- cluster.master(i)->bound_http_hostport().ToString(),
- page));
- }
- }
- for (int i = 0; i < cluster.num_tablet_servers(); i++) {
- for (const auto& page : ts_pages) {
- urls_.emplace_back(strings::Substitute(
- "http://$0$1",
- cluster.tablet_server(i)->bound_http_hostport().ToString(),
- page));
- }
- }
- std::random_device rdev;
- std::mt19937 gen(rdev());
- std::shuffle(urls_.begin(), urls_.end(), gen);
- CHECK_OK(Thread::Create("linked_list-test", "checker",
- &PeriodicWebUIChecker::CheckThread, this, &checker_));
- }
-
- ~PeriodicWebUIChecker() {
- LOG(INFO) << "Shutting down curl thread";
- is_running_ = false;
- if (checker_) {
- checker_->Join();
- }
- }
-
- private:
- void CheckThread() {
- EasyCurl curl;
- // Set some timeout so that if the page deadlocks, we fail the test.
- curl.set_timeout(MonoDelta::FromSeconds(120));
- faststring dst;
- LOG(INFO) << strings::Substitute(
- "curl thread will poll the following URLs every $0", period_.ToString());
- for (const auto& url : urls_) {
- LOG(INFO) << url;
- }
- while (is_running_) {
- // Poll all of the URLs.
- const MonoTime start = MonoTime::Now();
- bool compression_enabled = true;
- for (const auto& url : urls_) {
- // Switch compression back and forth.
- Status s = compression_enabled ? curl.FetchURL(url, &dst, {"Accept-Encoding: gzip"})
- : curl.FetchURL(url, &dst);
- compression_enabled = !compression_enabled;
- if (s.ok()) {
- CHECK_GT(dst.length(), 0);
- }
- CHECK(!s.IsTimedOut()) << strings::Substitute(
- "could not fetch $0 ($1 compression, $2 connections): $3",
- url, compression_enabled ? "gzip" : "no", curl.num_connects(), s.ToString());
- }
- // Sleep until the next period
- const MonoDelta elapsed = MonoTime::Now() - start;
- const int64_t sleep_ns = period_.ToNanoseconds() - elapsed.ToNanoseconds();
- if (sleep_ns > 0) {
- SleepFor(MonoDelta::FromNanoseconds(sleep_ns));
- }
- }
- }
-
- const MonoDelta period_;
- std::atomic<bool> is_running_;
- scoped_refptr<Thread> checker_;
- std::vector<std::string> urls_;
-};
-
// Helper class to hold results from a linked list scan and perform the
// verification step on the data.
class LinkedListVerifier {
diff --git a/src/kudu/integration-tests/linked_list-test.cc b/src/kudu/integration-tests/linked_list-test.cc
index eddc34c..fb4918c 100644
--- a/src/kudu/integration-tests/linked_list-test.cc
+++ b/src/kudu/integration-tests/linked_list-test.cc
@@ -39,7 +39,6 @@
#include <boost/bind.hpp>
#include <gflags/gflags.h>
-#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
@@ -53,6 +52,7 @@
#include "kudu/master/master.pb.h"
#include "kudu/mini-cluster/external_mini_cluster.h"
#include "kudu/mini-cluster/mini_cluster.h"
+#include "kudu/mini-cluster/webui_checker.h"
#include "kudu/tserver/tablet_server-test-base.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
diff --git a/src/kudu/integration-tests/webserver-stress-itest.cc b/src/kudu/integration-tests/webserver-stress-itest.cc
index 8a1c31f..74fa5ee 100644
--- a/src/kudu/integration-tests/webserver-stress-itest.cc
+++ b/src/kudu/integration-tests/webserver-stress-itest.cc
@@ -23,9 +23,9 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "kudu/integration-tests/linked_list-test-util.h"
#include "kudu/integration-tests/test_workload.h"
#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/mini-cluster/webui_checker.h"
#include "kudu/util/monotime.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
diff --git a/src/kudu/mini-cluster/CMakeLists.txt b/src/kudu/mini-cluster/CMakeLists.txt
index 9b4e55b..63f6eea 100644
--- a/src/kudu/mini-cluster/CMakeLists.txt
+++ b/src/kudu/mini-cluster/CMakeLists.txt
@@ -19,6 +19,7 @@ set(MINI_CLUSTER_SRCS
external_mini_cluster.cc
internal_mini_cluster.cc
mini_cluster.cc
+ webui_checker.cc
)
set(MINI_CLUSTER_LIBS
diff --git a/src/kudu/mini-cluster/webui_checker.cc b/src/kudu/mini-cluster/webui_checker.cc
new file mode 100644
index 0000000..0d97bbb
--- /dev/null
+++ b/src/kudu/mini-cluster/webui_checker.cc
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/mini-cluster/webui_checker.h"
+
+#include <algorithm>
+#include <atomic>
+#include <cstdint>
+#include <iostream>
+#include <random>
+#include <string>
+#include <vector>
+
+#include <glog/logging.h>
+
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/util/curl_util.h"
+#include "kudu/util/faststring.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/status.h"
+#include "kudu/util/thread.h"
+
+using std::string;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+
+PeriodicWebUIChecker::PeriodicWebUIChecker(
+ const cluster::ExternalMiniCluster& cluster,
+ MonoDelta period,
+ const string& tablet_id,
+ const vector<string>& master_pages,
+ const vector<string>& ts_pages)
+ : period_(period),
+ is_running_(true) {
+ // List of tserver web pages to fetch.
+ vector<string> ts_pages_spec(ts_pages);
+ if (tablet_id.empty()) {
+ ts_pages_spec.emplace_back("/transactions");
+ } else {
+ ts_pages_spec.emplace_back(
+ Substitute("/transactions?tablet_id=$0", tablet_id));
+ }
+
+ // Generate list of urls for each master and tablet server
+ for (int i = 0; i < cluster.num_masters(); i++) {
+ for (const auto& page : master_pages) {
+ urls_.emplace_back(Substitute(
+ "http://$0$1",
+ cluster.master(i)->bound_http_hostport().ToString(),
+ page));
+ }
+ }
+ for (int i = 0; i < cluster.num_tablet_servers(); i++) {
+ for (const auto& page : ts_pages_spec) {
+ urls_.emplace_back(Substitute(
+ "http://$0$1",
+ cluster.tablet_server(i)->bound_http_hostport().ToString(),
+ page));
+ }
+ }
+ std::random_device rdev;
+ std::mt19937 gen(rdev());
+ std::shuffle(urls_.begin(), urls_.end(), gen);
+ CHECK_OK(Thread::Create("test", "webui-checker",
+ &PeriodicWebUIChecker::CheckThread, this, &checker_));
+}
+
+PeriodicWebUIChecker::~PeriodicWebUIChecker() {
+ is_running_ = false;
+ if (checker_) {
+ LOG(INFO) << "shutting down CURL thread";
+ checker_->Join();
+ }
+}
+
+void PeriodicWebUIChecker::CheckThread() {
+ EasyCurl curl;
+ curl.set_timeout(MonoDelta::FromSeconds(120));
+ std::ostringstream ostr;
+ ostr << "curl thread will poll the following URLs every "
+ << period_.ToString() << ":\n";
+ for (const auto& url : urls_) {
+ ostr << " " << url << "\n";
+ }
+ LOG(INFO) << ostr.str();
+ faststring dst;
+ while (is_running_) {
+ // Poll all of the URLs.
+ const MonoTime start = MonoTime::Now();
+ bool compression_enabled = true;
+ for (const auto& url : urls_) {
+ // Switch compression back and forth.
+ Status s = compression_enabled
+ ? curl.FetchURL(url, &dst, {"Accept-Encoding: gzip"})
+ : curl.FetchURL(url, &dst);
+ compression_enabled = !compression_enabled;
+ if (s.ok()) {
+ CHECK_GT(dst.length(), 0);
+ }
+ CHECK(!s.IsTimedOut()) << Substitute(
+ "could not fetch $0 ($1 compression, $2 connections): $3",
+ url, compression_enabled ? "gzip" : "no", curl.num_connects(), s.ToString());
+ }
+ // Sleep until the next period
+ const MonoDelta elapsed = MonoTime::Now() - start;
+ const int64_t sleep_ns = period_.ToNanoseconds() - elapsed.ToNanoseconds();
+ if (sleep_ns > 0) {
+ SleepFor(MonoDelta::FromNanoseconds(sleep_ns));
+ }
+ }
+}
+
+} // namespace kudu
diff --git a/src/kudu/mini-cluster/webui_checker.h b/src/kudu/mini-cluster/webui_checker.h
new file mode 100644
index 0000000..6127f13
--- /dev/null
+++ b/src/kudu/mini-cluster/webui_checker.h
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <atomic>
+#include <string>
+#include <vector>
+
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/util/monotime.h"
+
+namespace kudu {
+class Thread;
+namespace cluster {
+class ExternalMiniCluster;
+} // namespace cluster
+
+// A task that periodically checks master's tablet server's Web UI pages.
+class PeriodicWebUIChecker {
+ public:
+ PeriodicWebUIChecker(const cluster::ExternalMiniCluster& cluster,
+ MonoDelta period,
+ const std::string& tablet_id = "",
+ const std::vector<std::string>& master_pages = {
+ "/dump-entities",
+ "/masters",
+ "/mem-trackers",
+ "/metrics",
+ "/stacks",
+ "/tables",
+ "/tablet-servers",
+ "/threadz",
+ "/threadz?group=all", },
+ const std::vector<std::string>& ts_pages = {
+ "/maintenance-manager",
+ "/mem-trackers",
+ "/metrics",
+ "/scans",
+ "/stacks",
+ "/tablets",
+ "/threadz",
+ "/threadz?group=all",
+ });
+
+ ~PeriodicWebUIChecker();
+
+ private:
+ void CheckThread();
+
+ const MonoDelta period_;
+ std::atomic<bool> is_running_;
+ scoped_refptr<Thread> checker_;
+ std::vector<std::string> urls_;
+};
+
+} // namespace kudu