You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by aw...@apache.org on 2021/11/30 05:51:03 UTC
[kudu] 02/02: [dense_node-itest] enable on macOS
This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit cac9b48ccee922b5d88cac20f496cd77b409f0bc
Author: Andrew Wong <aw...@cloudera.com>
AuthorDate: Thu Nov 18 18:38:43 2021 -0800
[dense_node-itest] enable on macOS
This patch addresses a TODO to enable this test on macOS once the file
cache is used for WAL segments. This occurred in commit
fc4ab691a502067bc4d5bdff30507cac7feb7cfe.
Since macOS doesn't currently support the LogBlockManager, this also
updates the test to look for some BlockManager-agnostic metrics if not
using the LogBlockManager.
I also found we can add more block-related stress to the test scenario
by disabling compactions, which I've done optionally (though leaving the
default behavior the same).
Change-Id: I240331c4c9293d4f449d52b6a4ed2392a6ec9f3c
Reviewed-on: http://gerrit.cloudera.org:8080/18055
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
src/kudu/integration-tests/CMakeLists.txt | 5 +---
src/kudu/integration-tests/dense_node-itest.cc | 36 +++++++++++++++++++++-----
2 files changed, 30 insertions(+), 11 deletions(-)
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index c687930..9e2f8ad 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -76,6 +76,7 @@ ADD_KUDU_TEST(create-table-stress-test RUN_SERIAL true)
ADD_KUDU_TEST(decimal-itest)
ADD_KUDU_TEST(delete_table-itest NUM_SHARDS 8 PROCESSORS 4)
ADD_KUDU_TEST(delete_tablet-itest PROCESSORS 2)
+ADD_KUDU_TEST(dense_node-itest RUN_SERIAL true)
ADD_KUDU_TEST(disk_failure-itest PROCESSORS 2)
ADD_KUDU_TEST(disk_reservation-itest)
ADD_KUDU_TEST(dns_alias-itest)
@@ -151,10 +152,6 @@ ADD_KUDU_TEST(write_throttling-itest)
if (NOT APPLE)
ADD_KUDU_TEST(minidump_generation-itest)
- # Opens too many files on macOS because we're not using a file cache for
- # reading log segments.
- # TODO(wdb): Re-enable once a file cache is being used.
- ADD_KUDU_TEST(dense_node-itest RUN_SERIAL true)
endif()
# Tests that should not be run automatically by ctest.
diff --git a/src/kudu/integration-tests/dense_node-itest.cc b/src/kudu/integration-tests/dense_node-itest.cc
index b75cd09..2e37625 100644
--- a/src/kudu/integration-tests/dense_node-itest.cc
+++ b/src/kudu/integration-tests/dense_node-itest.cc
@@ -19,7 +19,6 @@
#include <algorithm>
#include <cstdint>
-#include <initializer_list>
#include <memory>
#include <ostream>
#include <string>
@@ -31,6 +30,7 @@
#include <gtest/gtest.h>
#include "kudu/client/schema.h"
+#include "kudu/gutil/integral_types.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/integration-tests/cluster_itest_util.h"
#include "kudu/integration-tests/external_mini_cluster-itest-base.h"
@@ -44,12 +44,18 @@
#include "kudu/util/test_macros.h"
METRIC_DECLARE_entity(server);
+METRIC_DECLARE_gauge_uint64(block_manager_total_blocks_created);
+METRIC_DECLARE_gauge_uint64(block_manager_total_bytes_written);
+METRIC_DECLARE_gauge_uint64(block_manager_total_disk_sync);
METRIC_DECLARE_gauge_uint64(log_block_manager_blocks_under_management);
METRIC_DECLARE_gauge_uint64(log_block_manager_bytes_under_management);
METRIC_DECLARE_gauge_uint64(log_block_manager_containers);
METRIC_DECLARE_gauge_uint64(log_block_manager_full_containers);
METRIC_DECLARE_gauge_uint64(threads_running);
+DECLARE_bool(enable_rowset_compaction);
+DECLARE_string(block_manager);
+
DEFINE_bool(measure_startup_drop_caches, false,
"Whether to drop kernel caches before measuring startup time. Must be root");
DEFINE_bool(measure_startup_sync, false,
@@ -109,9 +115,15 @@ TEST_F(DenseNodeTest, RunTest) {
FLAGS_max_blocks_per_container),
// UNDO delta block GC runs a lot to eagerly open newly created cfiles.
- // Disable it so we can maximize flushes and compactions.
+ // Disable it so we can maximize flushes.
"--enable_undo_delta_block_gc=false",
+ // Compactions aim to reduce the number of rowsets and thus reduce the
+ // number of cfiles. Potentially disable them so we can maximize the
+ // number of files. NOTE: this also means we don't exercise the block
+ // deletions that come with compactions.
+ Substitute("--enable_rowset_compaction=$0", FLAGS_enable_rowset_compaction),
+
// Allow our single tserver to service many, many RPCs.
"--rpc_service_queue_length=1000",
@@ -177,11 +189,21 @@ TEST_F(DenseNodeTest, RunTest) {
// Collect some interesting metrics. The cluster is shut down before the
// metrics are logged so that they're easier to find in the log output.
vector<pair<string, int64_t>> metrics;
- for (const auto* m : { &METRIC_log_block_manager_blocks_under_management,
- &METRIC_log_block_manager_bytes_under_management,
- &METRIC_log_block_manager_containers,
- &METRIC_log_block_manager_full_containers,
- &METRIC_threads_running }) {
+ vector<GaugePrototype<uint64>*> metric_prototypes;
+ if (FLAGS_block_manager == "log") {
+ metric_prototypes = { &METRIC_log_block_manager_blocks_under_management,
+ &METRIC_log_block_manager_bytes_under_management,
+ &METRIC_log_block_manager_containers,
+ &METRIC_log_block_manager_full_containers,
+ &METRIC_block_manager_total_disk_sync,
+ &METRIC_threads_running };
+ } else {
+ metric_prototypes = { &METRIC_block_manager_total_bytes_written,
+ &METRIC_block_manager_total_blocks_created,
+ &METRIC_block_manager_total_disk_sync,
+ &METRIC_threads_running };
+ }
+ for (const auto* m : metric_prototypes) {
int64_t value;
ASSERT_OK(itest::GetInt64Metric(
cluster_->tablet_server(0)->bound_http_hostport(), &METRIC_ENTITY_server,