You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by ma...@apache.org on 2026/10/04 22:52:45 UTC
(trafficserver) branch master updated: cache: keep the stripe locked after the shutdown dir sync (#13767)
This is an automated email from the ASF dual-hosted git repository.
masaori335 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 2b65f92e32 cache: keep the stripe locked after the shutdown dir sync (#13767)
2b65f92e32 is described below
commit 2b65f92e32a5d0d52eb8c64fba76fe5b522969b2
Author: Masaori Koshiba <ma...@apache.org>
AuthorDate: Mon Oct 5 07:52:39 2026 +0900
cache: keep the stripe locked after the shutdown dir sync (#13767)
d039375185 made StripeSM::shutdown() release the stripe lock to avoid
a deadlock on a second shutdown pass. That let writers admitted after
the snapshot insert Dir entries for data that only reached the
aggregation buffer. The shm directory carries those entries into the
next start, where they point at whatever the new process writes there.
Hold the lock until exit again. Writers, readers and any AIO
completion for an already-synced stripe wait on the lock until exit.
Overlapping shutdown passes can deadlock on stripes of different
disks, but neither marks the directory clean; a separate change runs
the shutdown only once.
---
src/iocore/cache/StripeSM.cc | 12 +-
.../cache/unit_tests/test_CacheShmShutdown.cc | 126 ++++++++++++++++++++-
2 files changed, 129 insertions(+), 9 deletions(-)
diff --git a/src/iocore/cache/StripeSM.cc b/src/iocore/cache/StripeSM.cc
index d26813f8be..ef502563dd 100644
--- a/src/iocore/cache/StripeSM.cc
+++ b/src/iocore/cache/StripeSM.cc
@@ -1338,10 +1338,8 @@ StripeSM::add_writer(CacheVC *vc)
void
StripeSM::shutdown(EThread *shutdown_thread)
{
- // the process is going down, do a blocking call
- // dont release the volume's lock, there could
- // be another aggWrite in progress
- SCOPED_MUTEX_LOCK(lock, this->mutex, shutdown_thread);
+ // Never released, so no writer can add Dir entries after this sync.
+ MUTEX_TAKE_LOCK(this->mutex, shutdown_thread);
if (DISK_BAD(this->disk)) {
Dbg(dbg_ctl_cache_dir_sync, "Dir %s: bad disk -- invalidating shm copy for disk recovery", this->hash_text.get());
@@ -1349,8 +1347,7 @@ StripeSM::shutdown(EThread *shutdown_thread)
return;
}
- // aggWriteDone advances write_pos again once we drop the mutex, so the shm header is not final; the on-disk write below
- // plus recover_data() next start reconcile it.
+ // The shm header is not final while an AIO is in flight.
if (CacheShm::is_shm_pointer(this->directory.raw_dir) && this->is_io_in_progress()) {
Dbg(dbg_ctl_cache_dir_sync, "Dir %s: AIO write in flight -- invalidating shm copy, syncing dir to disk", this->hash_text.get());
CacheShm::invalidate_stripe_directory(this->directory.raw_dir);
@@ -1370,8 +1367,7 @@ StripeSM::shutdown(EThread *shutdown_thread)
if (!this->_write_buffer.is_empty()) {
Dbg(dbg_ctl_cache_dir_sync, "Dir %s: flushing agg buffer first", this->hash_text.get());
if (!this->flush_aggregate_write_buffer(this->fd)) {
- // Mark rather than lean on the unquiesced cursor the failure leaves behind: the event system is still up, so a later
- // aggWriteDone or agg_wrap() can re-equalize agg_pos and write_pos and the gate would let this segment through.
+ // Don't rely on the attach gate to reject the cursor this failure leaves.
Error("Dir %s: aggregation buffer flush failed during shutdown; syncing the directory to disk", this->hash_text.get());
CacheShm::invalidate_stripe_directory(this->directory.raw_dir);
}
diff --git a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc
index 56cff94692..f6e8b028aa 100644
--- a/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc
+++ b/src/iocore/cache/unit_tests/test_CacheShmShutdown.cc
@@ -32,6 +32,7 @@
#include "iocore/cache/Store.h"
#include "records/RecCore.h"
+#include "tsutil/PostScript.h"
#include <cstdio>
#include <string>
@@ -171,9 +172,11 @@ TEST_CASE("StripeSM::shutdown marks the stripe untrusted without touching the di
SCOPED_MUTEX_LOCK(lock, stripe.mutex, this_ethread());
stripe.shutdown(this_ethread());
}
+ // shutdown() keeps the stripe locked until process exit, and a held mutex cannot be destroyed with the stripe.
+ MUTEX_UNTAKE_LOCK(stripe.mutex, this_ethread());
// The mark goes to the control segment; raw_dir -- which is also the pwrite source -- must come through untouched, so
- // the disk copy stays loadable. shutdown() leaves the stripe locked, so read the file unlocked.
+ // the disk copy stays loadable.
CHECK(stripe_marked_untrusted(0));
CHECK(stripe.directory.header->magic == STRIPE_MAGIC);
@@ -273,6 +276,8 @@ TEST_CASE("StripeSM::shutdown marks a bad disk's stripe without writing to it",
SCOPED_MUTEX_LOCK(lock, stripe.mutex, this_ethread());
stripe.shutdown(this_ethread());
}
+ // shutdown() keeps the stripe locked until process exit, and a held mutex cannot be destroyed with the stripe.
+ MUTEX_UNTAKE_LOCK(stripe.mutex, this_ethread());
CHECK(stripe_marked_untrusted(0));
CHECK(stripe.directory.header->magic == STRIPE_MAGIC);
@@ -567,6 +572,8 @@ TEST_CASE("StripeSM::shutdown syncs the directory when the agg flush fails", "[c
SCOPED_MUTEX_LOCK(lock, stripe.mutex, this_ethread());
stripe.shutdown(this_ethread());
}
+ // shutdown() keeps the stripe locked until process exit, and a held mutex cannot be destroyed with the stripe.
+ MUTEX_UNTAKE_LOCK(stripe.mutex, this_ethread());
// Marked like the other two paths that cannot vouch for the directory; the cursor it leaves unquiesced is not durable.
CHECK(stripe.directory.header->write_pos == write_pos_before);
@@ -591,3 +598,120 @@ TEST_CASE("StripeSM::shutdown syncs the directory when the agg flush fails", "[c
unlink_test_segments();
}
+
+namespace
+{
+
+// Entries recover_data() would clear: in phase, at or past write_pos.
+int
+count_unflushed_entries(const Stripe &stripe)
+{
+ int count = 0;
+ for (int i = 0; i < stripe.directory.entries(); i++) {
+ const Dir *e = &stripe.directory.dir[i];
+ if (!dir_is_empty(e) && dir_phase(e) == stripe.directory.header->phase &&
+ stripe.vol_offset(e) >= stripe.directory.header->write_pos) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+} // namespace
+
+// shutdown() must keep the stripe locked, or a late writer's Dir entry outlives its aggregation buffer into the next start.
+TEST_CASE("An entry inserted after the shutdown snapshot does not survive the next attach", "[cache][shm]")
+{
+ // A clean stripe takes shutdown()'s early return, which must keep the lock too.
+ const bool dirty = GENERATE(true, false);
+ INFO("dirty " << dirty);
+
+ unlink_test_segments();
+ if (!enable_shm()) {
+ WARN("shm unavailable in this environment; skipping");
+ unlink_test_segments();
+ return;
+ }
+
+ CacheDisk disk;
+ init_disk(disk);
+ disk.hw_sector_size = 512;
+ CacheVol cache_vol;
+ off_t snapshot_write_pos{};
+
+ {
+ GateStripe stripe{&disk, 10, 0};
+ stripe.cache_vol = &cache_vol;
+ attach_tmpfile_to_stripe(stripe);
+ REQUIRE(CacheShm::is_shm_pointer(stripe.directory.raw_dir));
+
+ // Step 1. shutdown
+ stripe.clear_dir();
+ stripe.set_io_not_in_progress();
+ stripe.directory.header->dirty = dirty;
+
+ // A sentinel, as sync_cache_dir_on_shutdown() uses, so the test thread does not hold the lock.
+ char shutdown_sentinel;
+ EThread *shutdown_thread = reinterpret_cast<EThread *>(&shutdown_sentinel);
+ // Stands in for process exit, even when a REQUIRE unwinds; a held mutex can't be destroyed.
+ ts::PostScript release([&]() -> void { MUTEX_UNTAKE_LOCK(stripe.mutex, shutdown_thread); });
+ stripe.shutdown(shutdown_thread);
+ snapshot_write_pos = stripe.directory.header->write_pos;
+ REQUIRE(count_unflushed_entries(stripe) == 0);
+
+ // Step 2. a writer arrives after shutdown
+ cache_rsb.write_bytes = ts::Metrics::Counter::createPtr("unit_test.write.bytes");
+ cache_vol.vol_rsb.write_bytes = ts::Metrics::Counter::createPtr("unit_test.write.bytes");
+ cache_rsb.direntries_used = ts::Metrics::Gauge::createPtr("unit_test.direntries.used");
+ cache_vol.vol_rsb.direntries_used = ts::Metrics::Gauge::createPtr("unit_test.direntries.used");
+
+ CacheKey key;
+ key.b[0] = 0x1234;
+ key.b[1] = 0x5678;
+
+ WaitingVC writer{&stripe};
+ writer.key = key;
+ writer.first_key = key;
+ writer.set_test_data("yay", 4);
+ writer.set_write_len(4);
+ writer.set_agg_len(stripe.round_to_approx_size(writer.write_len + writer.header_len + writer.frag_len));
+
+ // Writers try-lock the stripe and retry on a miss.
+ {
+ MUTEX_TRY_LOCK(lock, stripe.mutex, this_ethread());
+ CHECK_FALSE(lock.is_locked());
+ // If admitted anyway, finish the write so Step 3 shows the entry it leaves.
+ if (lock.is_locked()) {
+ stripe.add_writer(&writer);
+ stripe.aggWrite(EVENT_NONE, nullptr);
+ writer.wait_for_callback();
+ REQUIRE(stripe.get_agg_buf_pos() > 0);
+
+ // The writer's callback inserts once its Doc is in the buffer, with no AIO issued yet.
+ REQUIRE(stripe.directory.insert(&key, &stripe, &writer.dir) == 1);
+ REQUIRE(count_unflushed_entries(stripe) == 1);
+ }
+ }
+
+ REQUIRE(stripe.directory.header->agg_pos == stripe.directory.header->write_pos);
+ CacheShm::mark_clean_shutdown();
+ }
+
+ // Step 3. restart and attach the same segment
+ CacheShm::release_for_test();
+ REQUIRE(enable_shm());
+ REQUIRE(CacheShm::mode() == CacheShm::Mode::AttachExisting);
+
+ {
+ GateStripe stripe{&disk, 10, 0};
+ stripe.cache_vol = &cache_vol;
+ REQUIRE(CacheShm::is_shm_pointer(stripe.directory.raw_dir));
+ REQUIRE(stripe.directory.header->magic == STRIPE_MAGIC);
+ REQUIRE(stripe.directory.header->write_pos == snapshot_write_pos);
+ REQUIRE(stripe.shm_directory_is_valid());
+
+ CHECK(count_unflushed_entries(stripe) == 0);
+ }
+
+ unlink_test_segments();
+}