You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/07 09:47:03 UTC
(iggy) 08/10: perf(server): keep filesystem work off shard executors
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch perf/task-per-partition
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit c47919f823339c261a52a7744a3fad7c3bd60eec
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Wed Oct 7 10:06:17 2026 +0200
perf(server): keep filesystem work off shard executors
Synchronous filesystem calls in metadata checkpoints, partition
recovery and statistics can stall every task on a shard.
Use Compio I/O and the bounded filesystem worker, retain cancellation
ordering, cache statistics and transfer offers, and sync dirty offset
directories. Preserve task-per-partition jobs and recovery barriers.
Capture metadata replay protection after snapshot I/O so concurrent
commits retain their sessions. Extend the regression to cover a
transferred table ahead of the local applied frontier.
(cherry picked from commit 202aefd9a28f551681cb0e3119851d2ac96a5559)
---
Cargo.lock | 1 +
core/journal/src/durable_storage.rs | 43 +--
core/journal/src/file_storage.rs | 37 +--
core/journal/src/partition_journal/segments.rs | 9 +-
core/journal/src/prepare_journal.rs | 241 +++++++++++++----
core/journal/src/superblock.rs | 49 ++--
core/metadata/Cargo.toml | 4 +-
core/metadata/src/impls/metadata.rs | 339 ++++++++++++++++++------
core/metadata/src/impls/recovery.rs | 30 ++-
core/partitions/src/iggy_partition.rs | 32 ++-
core/partitions/src/messages_writer.rs | 2 +-
core/partitions/src/persistence.rs | 25 +-
core/partitions/src/segment_recovery.rs | 351 ++++++++++++++-----------
core/partitions/src/state_transfer.rs | 23 +-
core/server/Cargo.toml | 1 +
core/server/src/boot/mod.rs | 4 +-
core/server/src/partition_helpers.rs | 45 +++-
core/server/src/partition_reconciler.rs | 2 +-
core/server/src/responses.rs | 7 +-
core/server/src/sysinfo_printer.rs | 20 +-
core/server/src/sysinfo_probe.rs | 159 +++++------
core/server_common/src/fs_utils.rs | 121 ++++++++-
core/server_common/src/segment_storage/mod.rs | 2 +-
core/shard/src/lib.rs | 2 +-
24 files changed, 1017 insertions(+), 532 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 961adcb98..4efd4507b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12705,6 +12705,7 @@ name = "server"
version = "0.9.1-edge.1"
dependencies = [
"ahash",
+ "arc-swap",
"argon2",
"assert_cmd",
"async-channel",
diff --git a/core/journal/src/durable_storage.rs b/core/journal/src/durable_storage.rs
index 45ce7d432..611dc8555 100644
--- a/core/journal/src/durable_storage.rs
+++ b/core/journal/src/durable_storage.rs
@@ -20,10 +20,9 @@
use compio::buf::{IntoInner, IoBuf};
use compio::fs::{File, OpenOptions};
use compio::io::{AsyncReadAtExt, AsyncWriteAtExt};
-use futures::channel::oneshot;
-use futures::lock::Mutex;
use futures::{Stream, stream};
use server_common::fatal::NoteDescriptorExhaustion;
+use server_common::fs_utils::{run_blocking, truncate_file};
use server_common::iobuf::{Frozen, Owned};
use std::ffi::OsString;
use std::io;
@@ -134,7 +133,9 @@ pub trait DurableStorage {
pub trait DurableFile {
/// Best-effort reservation that must not change bytes or logical length.
/// Backends without physical allocation may ignore this hint.
- fn preallocate(&self, _path: &Path, _length: u64) {}
+ fn preallocate(&self, _path: &Path, _length: u64) -> impl Future<Output = ()> {
+ async {}
+ }
/// # Errors
/// Returns an error if the complete range cannot be read.
@@ -394,8 +395,8 @@ impl DurableStorage for DiskStorage {
}
impl DurableFile for File {
- fn preallocate(&self, path: &Path, length: u64) {
- server_common::fs_utils::preallocate_file(self, path, length);
+ async fn preallocate(&self, path: &Path, length: u64) {
+ server_common::fs_utils::preallocate_file(self, path, length).await;
}
async fn read(&self, offset: u64, length: usize) -> io::Result<Vec<u8>> {
@@ -452,17 +453,7 @@ impl DurableFile for File {
}
async fn truncate(&self, length: u64) -> io::Result<()> {
- // Older kernels lack IORING_OP_FTRUNCATE and shard fallback pools are
- // disabled. Own the inode until the worker completes, even on cancellation.
- let descriptor = std::os::fd::AsFd::as_fd(self)
- .try_clone_to_owned()
- .note_descriptor_exhaustion(|| {
- "duplicating a file descriptor to truncate".to_owned()
- })?;
- run_blocking("iggy-file-truncate", move || {
- std::fs::File::from(descriptor).set_len(length)
- })
- .await
+ truncate_file(self, length).await
}
async fn sync(&self) -> io::Result<()> {
@@ -470,26 +461,6 @@ impl DurableFile for File {
}
}
-async fn run_blocking<T: Send + 'static>(
- name: &'static str,
- operation: impl FnOnce() -> io::Result<T> + Send + 'static,
-) -> io::Result<T> {
- // Keep the permit on the worker: cancelling its caller must not admit
- // another blocking operation while this one still owns filesystem state.
- static WORKER: Mutex<()> = Mutex::new(());
- let permit = WORKER.lock().await;
- let (sender, receiver) = oneshot::channel();
- std::thread::Builder::new()
- .name(name.to_owned())
- .spawn(move || {
- let _permit = permit;
- let _ = sender.send(operation());
- })?;
- receiver
- .await
- .map_err(|_| io::Error::other(format!("{name} stopped")))?
-}
-
fn directory_entries(path: &Path) -> io::Result<Vec<StorageEntry>> {
std::fs::read_dir(path)?
.map(|entry| {
diff --git a/core/journal/src/file_storage.rs b/core/journal/src/file_storage.rs
index 97061f7b7..7b9f23446 100644
--- a/core/journal/src/file_storage.rs
+++ b/core/journal/src/file_storage.rs
@@ -19,10 +19,9 @@ use crate::Storage;
use compio::buf::IoBuf;
use compio::io::{AsyncReadAtExt, AsyncWriteAtExt};
use server_common::fatal::NoteDescriptorExhaustion;
+use server_common::fs_utils::truncate_file;
use std::cell::{Cell, UnsafeCell};
-use std::fs;
use std::io;
-use std::os::fd::AsFd;
use std::path::{Path, PathBuf};
/// File-backed storage implementing the `Storage` trait.
@@ -63,34 +62,17 @@ impl FileStorage {
/// Truncate the file to `len` bytes and make the new length durable.
///
- /// Synchronous `std::fs` on a duplicate of the open descriptor, not compio:
- /// compio's `set_len` submits `IORING_OP_FTRUNCATE`, which landed in
- /// mainline Linux 6.9. When the opcode is unavailable, the driver falls
- /// back to its blocking pool, and shard proactors run with
- /// `thread_pool_limit(0)`, so the fallback panics the shard instead of
- /// repairing the WAL. `std::fs` needs neither the opcode nor the pool. The
- /// sole caller is boot-time torn-tail repair, so blocking the shard thread
- /// here costs nothing.
- ///
- /// `sync_all` makes the durable-truncation contract explicit and matches
- /// segment recovery. Its additional metadata synchronization is acceptable
- /// because this runs only during boot-time repair.
+ /// Uses the bounded filesystem worker for kernels without `IORING_OP_FTRUNCATE`,
+ /// then asynchronously syncs the inode before recovery can use the new boundary.
///
/// # Errors
/// Returns an I/O error if the descriptor cannot be cloned, truncated, or synced.
- pub(crate) fn truncate(&self, len: u64) -> io::Result<()> {
+ pub(crate) async fn truncate(&self, len: u64) -> io::Result<()> {
// SAFETY: single-threaded compio runtime, no concurrent access to the file.
let file = unsafe { &*self.file.get() };
- let file = fs::File::from(
- file.as_fd()
- .try_clone_to_owned()
- .note_descriptor_exhaustion(|| {
- format!("duplicating the descriptor of {}", self.path.display())
- })?,
- );
- file.set_len(len)?;
+ truncate_file(file, len).await?;
self.write_offset.set(len);
- file.sync_all()
+ file.sync_all().await
}
/// Fsync the file to disk.
@@ -206,9 +188,6 @@ mod tests {
use server_common::executor::create_shard_executor;
use tempfile::tempdir;
- /// Pins the synchronous truncate signature and verifies it works inside a
- /// shard executor with no blocking pool. A modern test kernel supports
- /// `IORING_OP_FTRUNCATE`, so this does not reproduce compio's fallback.
#[test]
fn given_a_shard_executor_with_no_blocking_pool_when_truncating_should_repair_the_file() {
let runtime = create_shard_executor().unwrap();
@@ -218,7 +197,7 @@ mod tests {
let storage = FileStorage::open(&path).await.unwrap();
storage.write_append(vec![0xAB_u8; 128]).await.unwrap();
- storage.truncate(64).unwrap();
+ storage.truncate(64).await.unwrap();
assert_eq!(storage.file_len(), 64);
assert_eq!(std::fs::metadata(&path).unwrap().len(), 64);
@@ -237,7 +216,7 @@ mod tests {
std::fs::rename(&path, &renamed_path).unwrap();
std::fs::write(&path, vec![0xCD_u8; 256]).unwrap();
- storage.truncate(64).unwrap();
+ storage.truncate(64).await.unwrap();
assert_eq!(storage.file_len(), 64);
assert_eq!(std::fs::metadata(&renamed_path).unwrap().len(), 64);
diff --git a/core/journal/src/partition_journal/segments.rs b/core/journal/src/partition_journal/segments.rs
index 44067c49d..31876026a 100644
--- a/core/journal/src/partition_journal/segments.rs
+++ b/core/journal/src/partition_journal/segments.rs
@@ -275,7 +275,9 @@ impl<S: DurableStorage> PartitionPrepareJournal<S> {
));
}
if self.preallocate_segments {
- installed_file.preallocate(&cursor.path(&self.directory), max_size);
+ installed_file
+ .preallocate(&cursor.path(&self.directory), max_size)
+ .await;
}
let checkpoint_prepare = if let Some(prepare) = prepare {
let header = self.validate_checkpoint_prepare(&prepare)?;
@@ -598,7 +600,8 @@ impl<S: DurableStorage> PartitionPrepareJournal<S> {
file.truncate(cursor.position.length).await?;
self.segment_files_dirty = true;
if self.preallocate_segments {
- file.preallocate(&cursor.path(&self.directory), segments.max_size);
+ file.preallocate(&cursor.path(&self.directory), segments.max_size)
+ .await;
}
}
self.sync_segment_files().await?;
@@ -716,7 +719,7 @@ impl<S: DurableStorage> PartitionPrepareJournal<S> {
if length == 0
&& let Some(size) = preallocate_size
{
- file.preallocate(&retained, size);
+ file.preallocate(&retained, size).await;
}
self.segment_files_dirty = true;
self.segment_links_dirty = true;
diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs
index be4013740..79cf37370 100644
--- a/core/journal/src/prepare_journal.rs
+++ b/core/journal/src/prepare_journal.rs
@@ -18,6 +18,8 @@
use crate::file_storage::FileStorage;
use crate::{Journal, JournalHandle};
use compio::io::AsyncWriteAtExt;
+use futures::channel::oneshot;
+use futures::lock::{Mutex, OwnedMutexGuard};
use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Command, PrepareHeader};
use server_common::fatal::NoteDescriptorExhaustion;
use server_common::{MESSAGE_ALIGN, Message, iobuf::Owned};
@@ -26,6 +28,8 @@ use std::fmt;
use std::io;
use std::ops::RangeInclusive;
use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
use twox_hash::XxHash3_64;
const HEADER_SIZE: usize = size_of::<PrepareHeader>();
@@ -152,13 +156,13 @@ pub struct PrepareJournal {
/// `RefCell` borrow-panic risk on the read fast path.
poisoned: OnceCell<PoisonState>,
/// True while a `drain()` is rewriting the WAL. Concurrent drains
- /// share the one `wal.tmp` swap and race it (truncated tmp, ENOENT
- /// on the losing rename, reads through a reopened fd), so overlap is
+ /// replace the same WAL and would race reads through the reopened fd, so overlap is
/// refused up front with `ResourceBusy` instead. The upper layer
/// serializes checkpoints anyway (metadata `journal_gate`); this is
/// the journal's own defense so no future caller can reintroduce the
/// race silently.
drain_in_flight: Cell<bool>,
+ temporary_file_gate: Arc<Mutex<()>>,
/// Number of slots in the in-memory index; ops map to `op % slot_count`.
/// [`DEFAULT_SLOT_COUNT`] unless the operator overrode it. Larger values
/// let more committed-but-unsnapshotted entries accumulate between
@@ -244,7 +248,7 @@ async fn truncate_or_fail(
reason,
"truncating torn WAL tail; no complete entry follows the damage"
);
- storage.truncate(pos)?;
+ storage.truncate(pos).await?;
Ok(())
}
@@ -315,39 +319,133 @@ fn valid_entry_header(scratch: &mut Owned<16>, bytes: &[u8]) -> Option<PrepareHe
Some(header)
}
-/// Best-effort unlink of a temp file on any error path between its
-/// `File::create` and the atomic `rename`. Without this, every failed
-/// write leaks a tmp file next to its target; the next attempt truncates
-/// it on re-create so safety holds, but operators see the tmp files
-/// accumulate across crashed/aborted writes. `defuse` is called after a
-/// successful rename so the now-renamed file is not removed. Shared with
-/// `superblock::atomic_replace`, which has the same window.
-///
-/// `Drop` cannot be async, so the unlink is a blocking `std::fs::remove_file`.
-/// This only runs on the drain failure path (already returning an error),
-/// so a sync syscall here is acceptable. Errors are swallowed: the file
-/// may already be gone (e.g. rename succeeded but a later step failed
-/// and we defused too late) and there is no useful recovery from a
-/// failed cleanup unlink.
+/// Each replacement owns a distinct scratch path, so delayed cancellation cleanup
+/// cannot unlink a later writer's file. Crash leftovers are swept when opening the store.
pub(crate) struct TmpFileGuard {
- path: PathBuf,
- armed: bool,
+ cleanup: Option<(PathBuf, OwnedMutexGuard<()>)>,
}
impl TmpFileGuard {
- pub(crate) const fn new(path: PathBuf) -> Self {
- Self { path, armed: true }
+ #[allow(clippy::future_not_send)]
+ pub(crate) async fn create(
+ base: &Path,
+ gate: &Arc<Mutex<()>>,
+ ) -> io::Result<(compio::fs::File, Self)> {
+ static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0);
+ let permit = Arc::clone(gate).lock_owned().await;
+ let sequence = NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed);
+ let mut path = base.as_os_str().to_owned();
+ path.push(format!(".{}-{sequence}", std::process::id()));
+ let path = PathBuf::from(path);
+ let (sender, receiver) = oneshot::channel();
+ // A cancelled open still owns cleanup until the kernel finishes creating the file.
+ compio::runtime::spawn(async move {
+ let guard = Self {
+ cleanup: Some((path.clone(), permit)),
+ };
+ let opened = compio::fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(&path)
+ .await
+ .note_descriptor_exhaustion(|| format!("creating {}", path.display()));
+ let result = match opened {
+ Ok(file) => Ok((file, guard)),
+ Err(error) => {
+ guard.cleanup().await;
+ Err(error)
+ }
+ };
+ let _ = sender.send(result);
+ })
+ .detach();
+ receiver
+ .await
+ .map_err(|_| io::Error::other("temporary file opener stopped"))?
+ }
+
+ pub(crate) fn path(&self) -> &Path {
+ &self
+ .cleanup
+ .as_ref()
+ .expect("temporary file guard is armed")
+ .0
}
pub(crate) fn defuse(mut self) {
- self.armed = false;
+ self.cleanup.take();
+ }
+
+ #[allow(clippy::future_not_send)]
+ pub(crate) async fn cleanup(mut self) {
+ if let Some((path, permit)) = self.cleanup.take() {
+ let _ = schedule_tmp_cleanup(path, permit).await;
+ }
}
}
impl Drop for TmpFileGuard {
fn drop(&mut self) {
- if self.armed {
- let _ = std::fs::remove_file(&self.path);
+ if let Some((path, permit)) = self.cleanup.take() {
+ drop(schedule_tmp_cleanup(path, permit));
+ }
+ }
+}
+
+fn schedule_tmp_cleanup(path: PathBuf, permit: OwnedMutexGuard<()>) -> oneshot::Receiver<()> {
+ let (sender, receiver) = oneshot::channel();
+ let _ = compio::runtime::Runtime::try_with_current(|_| {
+ compio::runtime::spawn(async move {
+ // A retry must wait for unlink and any older rename of this unique source.
+ let _permit = permit;
+ let _ = compio::fs::remove_file(path).await;
+ let _ = sender.send(());
+ })
+ .detach();
+ });
+ receiver
+}
+
+/// Called before a store admits writers; only its owned temporary-file names qualify.
+#[allow(clippy::future_not_send)]
+pub(crate) async fn sweep_tmp_files(bases: &[PathBuf]) {
+ let Some(base) = bases.first() else {
+ return;
+ };
+ let prefixes: Vec<_> = bases
+ .iter()
+ .filter_map(|base| {
+ let mut prefix = base.file_name()?.as_encoded_bytes().to_vec();
+ prefix.push(b'.');
+ Some(prefix)
+ })
+ .collect();
+ let directory = base
+ .parent()
+ .unwrap_or_else(|| Path::new("."))
+ .to_path_buf();
+ let scanned = server_common::fs_utils::run_blocking("iggy-temporary-scan", move || {
+ Ok(std::fs::read_dir(directory)?
+ .filter_map(Result::ok)
+ .filter(|entry| {
+ let name = entry.file_name();
+ prefixes.iter().any(|prefix| {
+ name.as_encoded_bytes()
+ .strip_prefix(prefix.as_slice())
+ .and_then(|suffix| std::str::from_utf8(suffix).ok())
+ .and_then(|suffix| suffix.split_once('-'))
+ .is_some_and(|(process, sequence)| {
+ process.parse::<u32>().is_ok() && sequence.parse::<u64>().is_ok()
+ })
+ })
+ })
+ .map(|entry| entry.path())
+ .collect::<Vec<_>>())
+ })
+ .await;
+ if let Ok(paths) = scanned {
+ for path in paths {
+ let _ = compio::fs::remove_file(path).await;
}
}
}
@@ -426,6 +524,7 @@ impl PrepareJournal {
"journal slot_count must be non-zero",
)));
}
+ sweep_tmp_files(&[path.with_extension("wal.tmp")]).await;
let storage = FileStorage::open(path).await?;
Self::scan(storage, snapshot_op, slot_count).await
}
@@ -637,6 +736,7 @@ impl PrepareJournal {
snapshot_op: Cell::new(snapshot_op),
poisoned: OnceCell::new(),
drain_in_flight: Cell::new(false),
+ temporary_file_gate: Arc::new(Mutex::new(())),
slot_count,
unsealed_entries,
})
@@ -772,11 +872,11 @@ impl PrepareJournal {
/// Replace the WAL with the `live` entries, in the given order, and point the
/// storage at the new file. The caller holds `drain_in_flight`, because every
- /// rewrite goes through the same tmp path, and rebuilds the index afterwards.
+ /// rewrite replaces the same WAL and rebuilds the index afterwards.
///
/// A failure before the rename leaves the old WAL in place and removes the tmp
- /// file. A failure after it poisons the journal under the matching name in
- /// `stages`.
+ /// file. Cancellation schedules cleanup of this attempt's scratch path.
+ /// A failure after rename poisons the journal under the matching name in `stages`.
#[allow(clippy::future_not_send)]
async fn rewrite_wal(
&self,
@@ -784,12 +884,12 @@ impl PrepareJournal {
stages: &RewriteStages,
) -> io::Result<()> {
let wal_path = self.storage.path();
- let tmp_path = wal_path.with_extension("wal.tmp");
- let tmp_guard = TmpFileGuard::new(tmp_path.clone());
- {
- let mut tmp = compio::fs::File::create(&tmp_path)
- .await
- .note_descriptor_exhaustion(|| format!("creating {}", tmp_path.display()))?;
+ let (mut tmp, guard) = TmpFileGuard::create(
+ &wal_path.with_extension("wal.tmp"),
+ &self.temporary_file_gate,
+ )
+ .await?;
+ let prepared = async {
let mut write_pos: u64 = 0;
for (header, old_offset) in live {
let size = header.size as usize;
@@ -800,7 +900,14 @@ impl PrepareJournal {
write_pos += size as u64;
}
tmp.sync_all().await?;
+ compio::fs::rename(guard.path(), wal_path).await
}
+ .await;
+ if let Err(error) = prepared {
+ guard.cleanup().await;
+ return Err(error);
+ }
+ guard.defuse();
// COMMIT POINT. From here on the on-disk WAL is the new one while the
// in-memory index still describes the old layout. Every fallible step
@@ -808,10 +915,6 @@ impl PrepareJournal {
// cannot write at a stale `write_offset` into the orphaned old fd, and
// the next `entry`/`entry_at` cannot serve offsets that no longer exist
// in the new file.
- compio::fs::rename(&tmp_path, wal_path).await?;
- // The rename consumed `tmp_path`, so nothing is left to unlink.
- tmp_guard.defuse();
-
// Without the parent directory fsync the rename can be lost across a
// power failure, and the old WAL comes back on recovery. With the
// journal poisoned the caller learns that the rewrite is not durable.
@@ -875,7 +978,7 @@ impl Journal for PrepareJournal {
"truncate_from: ops are 1-based, so 0 would discard the whole journal",
));
}
- // Shares the drain guard: both rewrite the same WAL through the same tmp
+ // Shares the drain guard: both rewrite the same WAL
// path, so letting them overlap would race the swap.
if self.drain_in_flight.replace(true) {
return Err(io::Error::new(
@@ -986,10 +1089,8 @@ impl Journal for PrepareJournal {
if let Some(state) = self.poisoned.get() {
return Err(Self::poisoned_io_error(state));
}
- // Overlapping drains race the `wal.tmp` swap below (truncate each
- // other's tmp mid-write, lose the rename to ENOENT, read stale
- // offsets through the winner's reopened fd). Refuse up front,
- // before any WAL bytes move.
+ // Overlapping drains would read stale offsets through the winner's
+ // reopened fd. Refuse them before any WAL bytes move.
if self.drain_in_flight.replace(true) {
return Err(io::Error::new(
io::ErrorKind::ResourceBusy,
@@ -1248,6 +1349,47 @@ mod tests {
(op as usize - 1) * (HEADER_SIZE + body_size) + field_offset
}
+ #[test]
+ fn given_cancelled_temporary_open_when_retrying_should_finish_cleanup_before_reuse() {
+ let runtime = server_common::executor::create_shard_executor().unwrap();
+ runtime.block_on(async {
+ let directory = tempfile::tempdir().unwrap();
+ let base = directory.path().join("journal.wal.tmp");
+ let gate = Arc::new(Mutex::new(()));
+ let mut cancelled = Box::pin(TmpFileGuard::create(&base, &gate));
+ assert!(futures::poll!(&mut cancelled).is_pending());
+ drop(cancelled);
+
+ let (file, guard) = TmpFileGuard::create(&base, &gate).await.unwrap();
+ let old_path = guard.path().to_path_buf();
+ assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1);
+ drop(file);
+ drop(guard);
+
+ let (file, replacement) = TmpFileGuard::create(&base, &gate).await.unwrap();
+ assert_ne!(replacement.path(), old_path);
+ assert!(!old_path.exists());
+ assert!(replacement.path().exists());
+ drop(file);
+ replacement.cleanup().await;
+ assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0);
+ });
+ }
+
+ #[compio::test]
+ async fn given_crash_scratch_when_opening_journal_should_remove_only_owned_temporary_files() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("journal.wal");
+ let orphan = directory.path().join("journal.wal.tmp.7-11");
+ let unrelated = directory.path().join("journal.wal.tmp.backup");
+ std::fs::write(&orphan, b"incomplete").unwrap();
+ std::fs::write(&unrelated, b"keep").unwrap();
+ let journal = PrepareJournal::open(&path, 0).await.unwrap();
+ assert_eq!(journal.last_op(), None);
+ assert!(!orphan.exists());
+ assert_eq!(std::fs::read(&unrelated).unwrap(), b"keep");
+ }
+
#[compio::test]
async fn truncate_from_removes_the_suffix_and_keeps_the_snapshot_floor() {
// The property that makes this not-a-drain: the floor must not move, or the
@@ -1798,7 +1940,7 @@ mod tests {
let storage = FileStorage::open(&path).await.unwrap();
let full_len = storage.file_len();
// Remove the last 10 bytes (partial second entry)
- storage.truncate(full_len - 10).unwrap();
+ storage.truncate(full_len - 10).await.unwrap();
}
// Reopen, should recover only the first entry
@@ -2394,15 +2536,8 @@ mod tests {
#[compio::test]
async fn concurrent_drains_are_refused_not_raced() {
- // Two drivers racing `drain()` share the one fixed `wal.tmp`:
- // `File::create` truncates the other racer's tmp mid-write, the
- // first rename consumes the path, and the loser surfaces ENOENT
- // (production: `forced checkpoint failed ... snapshot I/O error:
- // No such file or directory`) — or, with luckier timing, both
- // renames "succeed" over each other's bytes. Contract: exactly
- // one drain runs; a concurrent call is refused with
- // `ResourceBusy` before it touches the WAL; the journal stays
- // healthy either way.
+ // Both drains would replace the same WAL and rebuild its index.
+ // Exactly one may run; the other must leave the journal untouched.
let dir = tempdir().unwrap();
let path = dir.path().join("journal.wal");
let journal = PrepareJournal::open(&path, 0).await.unwrap();
@@ -2423,7 +2558,7 @@ mod tests {
assert_eq!(
losers[0].as_ref().unwrap_err().kind(),
io::ErrorKind::ResourceBusy,
- "loser must be refused up front, not fail mid-flight on the shared tmp: {losers:?}"
+ "loser must be refused before rewriting the WAL: {losers:?}"
);
// The journal must remain fully usable after the refused call.
diff --git a/core/journal/src/superblock.rs b/core/journal/src/superblock.rs
index a51fee1bb..8f9a9b6d7 100644
--- a/core/journal/src/superblock.rs
+++ b/core/journal/src/superblock.rs
@@ -52,7 +52,9 @@ use std::path::{Path, PathBuf};
use compio::io::{AsyncReadAtExt, AsyncWriteAtExt};
use server_common::fatal::NoteDescriptorExhaustion;
-use crate::prepare_journal::TmpFileGuard;
+use crate::prepare_journal::{TmpFileGuard, sweep_tmp_files};
+use futures::lock::Mutex;
+use std::sync::Arc;
use twox_hash::XxHash3_64;
/// Identifies a superblock file and rejects a foreign or zeroed one. "SBLK".
@@ -168,6 +170,7 @@ impl Slot {
/// Two-file ping-pong superblock. See the module docs for the durability
/// argument.
pub struct PingPongSuperblock {
+ temporary_file_gate: Arc<Mutex<()>>,
dir: PathBuf,
/// Sequence to stamp on the next `write`. Monotonic; selects the latest
/// record on read.
@@ -212,6 +215,7 @@ impl PingPongSuperblock {
dir: impl Into<PathBuf>,
) -> io::Result<(Self, SuperblockContents)> {
let dir = dir.into();
+ sweep_tmp_files(&[FILE_A, FILE_B].map(|name| dir.join(format!("{name}.tmp")))).await;
let slot_a = read_slot(&dir.join(FILE_A)).await?;
let slot_b = read_slot(&dir.join(FILE_B)).await?;
let seq_a = sequence_of(&slot_a);
@@ -233,6 +237,7 @@ impl PingPongSuperblock {
let next_slot = if a_is_newest { Slot::B } else { Slot::A };
let store = Self {
+ temporary_file_gate: Arc::new(Mutex::new(())),
dir,
next_sequence: Cell::new(latest + 1),
next_slot: Cell::new(next_slot),
@@ -272,7 +277,13 @@ impl SuperblockStore for PingPongSuperblock {
// contract nobody violated, sending whoever debugs it to audit the lock.
#[cfg(debug_assertions)]
let _writing = WritingGuard::acquire(&self.writing);
- atomic_replace(&self.dir, slot.file_name(), record).await?;
+ atomic_replace(
+ &self.dir,
+ slot.file_name(),
+ record,
+ &self.temporary_file_gate,
+ )
+ .await?;
self.next_sequence.set(sequence + 1);
self.next_slot.set(slot.other());
Ok(())
@@ -525,23 +536,27 @@ async fn read_slot(path: &Path) -> io::Result<SlotClass> {
/// Replace `dir/file_name` atomically: write a temp, fsync it, rename over the
/// target, then fsync the directory so the rename itself is durable.
-async fn atomic_replace(dir: &Path, file_name: &str, bytes: Vec<u8>) -> io::Result<()> {
- let tmp_path = dir.join(format!("{file_name}.tmp"));
+async fn atomic_replace(
+ dir: &Path,
+ file_name: &str,
+ bytes: Vec<u8>,
+ gate: &Arc<Mutex<()>>,
+) -> io::Result<()> {
+ let (mut tmp, guard) =
+ TmpFileGuard::create(&dir.join(format!("{file_name}.tmp")), gate).await?;
let final_path = dir.join(file_name);
- // Unlink the temp on any failure before the rename. Not a correctness matter,
- // since `File::create` truncates and `next_sequence` / `next_slot` stay
- // un-advanced so a retry re-targets the same slot; it keeps a failing disk from
- // littering `superblock.{a,b}.tmp` next to the slots an operator is inspecting.
- let guard = TmpFileGuard::new(tmp_path.clone());
- let mut tmp = compio::fs::File::create(&tmp_path)
- .await
- .note_descriptor_exhaustion(|| format!("creating {}", tmp_path.display()))?;
- let (result, _buf) = tmp.write_all_at(bytes, 0).await.into();
- result?;
- tmp.sync_all().await?;
-
- compio::fs::rename(&tmp_path, &final_path).await?;
+ let prepared = async {
+ let (result, _buf) = tmp.write_all_at(bytes, 0).await.into();
+ result?;
+ tmp.sync_all().await?;
+ compio::fs::rename(guard.path(), &final_path).await
+ }
+ .await;
+ if let Err(error) = prepared {
+ guard.cleanup().await;
+ return Err(error);
+ }
guard.defuse();
let dir_file = compio::fs::File::open(dir)
diff --git a/core/metadata/Cargo.toml b/core/metadata/Cargo.toml
index 5380ccc96..3caae88d4 100644
--- a/core/metadata/Cargo.toml
+++ b/core/metadata/Cargo.toml
@@ -41,7 +41,9 @@ simulator = []
ahash = { workspace = true }
bytemuck = { workspace = true }
bytes = { workspace = true }
+compio = { workspace = true }
consensus = { workspace = true }
+futures = { workspace = true }
iggy_binary_protocol = { workspace = true }
iggy_common = { workspace = true }
journal = { workspace = true }
@@ -56,8 +58,6 @@ tracing = { workspace = true }
[dev-dependencies]
bytes = { workspace = true }
-compio = { workspace = true }
-futures = { workspace = true }
tempfile = { workspace = true }
[lints.clippy]
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs
index f334a74d7..024719883 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -20,11 +20,12 @@ use crate::applied_frontier::AppliedFrontier;
use crate::stm::authz::{PartitionsCreate, admits_partitions_create, gated_apply};
use crate::stm::consumer_group::CompleteConsumerGroupRevocationRequest;
use crate::stm::snapshot::{
- FillSnapshot, MetadataSnapshot, RestoreSnapshotInPlace, Snapshot, SnapshotError,
+ FillSnapshot, MetadataSnapshot, PersistStage, RestoreSnapshotInPlace, Snapshot, SnapshotError,
};
use crate::stm::stream::{Streams, TruncatePartitionRequest};
use crate::stm::user::{DeletePersonalAccessTokenRequest, Users};
use crate::stm::{ConsensusGroupAllocator, StateMachine};
+use compio::io::AsyncWriteAtExt;
use consensus::{
CLIENTS_TABLE_MAX, Canceled, ClientTable, ClientTableSnapshot, CommitLogEvent, CommitReply,
Consensus, EXPIRED_SESSION_REQUEST_ID, EvictionContext, FatalReason, Pipeline, PipelineEntry,
@@ -37,6 +38,8 @@ use consensus::{
register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight,
send_eviction_to_client, send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity,
};
+use futures::channel::oneshot;
+use futures::lock::Mutex;
use iggy_binary_protocol::WireIdentifier;
use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment;
use iggy_binary_protocol::requests::partitions::CreatePartitionsRequest as WireCreatePartitionsRequest;
@@ -66,12 +69,16 @@ use server_common::Message;
use server_common::fatal::NoteDescriptorExhaustion;
use server_common::iobuf::{Frozen, Owned};
use std::cell::{Cell, RefCell};
+use std::io;
use std::mem::size_of;
use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
+// The writer retains this gate after caller cancellation, until the rename is durable.
+static SNAPSHOT_IO: Mutex<()> = Mutex::new(());
+
fn freeze_client_reply(
message: Message<GenericHeader>,
) -> server_common::iobuf::Frozen<{ server_common::MESSAGE_ALIGN }> {
@@ -126,8 +133,8 @@ impl IggySnapshot {
///
/// # Errors
/// Returns `SnapshotError` if serialization or I/O fails.
- pub fn persist(&self, path: &Path) -> Result<(), SnapshotError> {
- Self::write_durably(path, &self.encode()?)
+ pub async fn persist(&self, path: &Path) -> Result<(), SnapshotError> {
+ Self::write_durably(path, self.encode()?).await
}
/// Write already-encoded snapshot bytes to `path` durably: temp, fsync,
@@ -140,54 +147,64 @@ impl IggySnapshot {
///
/// # Errors
/// `SnapshotError::Persist` if any write, fsync, or rename fails.
- pub(super) fn write_durably(path: &Path, encoded: &[u8]) -> Result<(), SnapshotError> {
- use crate::stm::snapshot::PersistStage;
- use std::fs;
- use std::io::Write;
+ pub(super) async fn write_durably(path: &Path, encoded: Vec<u8>) -> Result<(), SnapshotError> {
+ let permit = SNAPSHOT_IO.lock().await;
+ let path = path.to_path_buf();
+ let (sender, receiver) = oneshot::channel();
+ compio::runtime::spawn(async move {
+ let _permit = permit;
+ let result = Self::write_durably_inner(&path, encoded).await;
+ let _ = sender.send(result);
+ })
+ .detach();
+ receiver
+ .await
+ .map_err(|_| SnapshotError::Io(io::Error::other("snapshot writer stopped")))?
+ }
+ #[allow(clippy::future_not_send)]
+ async fn write_durably_inner(path: &Path, mut encoded: Vec<u8>) -> Result<(), SnapshotError> {
+ let trailer = snapshot_trailer(&encoded);
+ encoded.extend_from_slice(&trailer);
let tmp_path = path.with_extension("bin.tmp");
- let mut file = fs::File::create(&tmp_path)
+ let mut file = compio::fs::File::create(&tmp_path)
+ .await
.note_descriptor_exhaustion(|| format!("creating {}", tmp_path.display()))
.map_err(|e| SnapshotError::Persist {
stage: PersistStage::Write,
source: e,
})?;
- file.write_all(encoded)
- .map_err(|e| SnapshotError::Persist {
- stage: PersistStage::Write,
- source: e,
- })?;
- // Self-verifying trailer. The superblock's checkpoint pairing cannot stand in:
- // phase 1 of a checkpoint renames the new snapshot over `snapshot.bin`, so a
- // crash before the pairing write is the NORMAL crash-inside-a-checkpoint
- // outcome, and it recovers through the `checkpoint_op < snapshot_op` arm, which
- // accepts the snapshot with nothing to check it against.
- file.write_all(&snapshot_trailer(encoded))
- .map_err(|e| SnapshotError::Persist {
+ file.write_all_at(encoded, 0)
+ .await
+ .0
+ .map_err(|source| SnapshotError::Persist {
stage: PersistStage::Write,
- source: e,
+ source,
})?;
- file.sync_all().map_err(|e| SnapshotError::Persist {
+ file.sync_all().await.map_err(|e| SnapshotError::Persist {
stage: PersistStage::Sync,
source: e,
})?;
drop(file);
- fs::rename(&tmp_path, path).map_err(|e| SnapshotError::Persist {
- stage: PersistStage::Rename,
- source: e,
- })?;
+ compio::fs::rename(&tmp_path, path)
+ .await
+ .map_err(|e| SnapshotError::Persist {
+ stage: PersistStage::Rename,
+ source: e,
+ })?;
// Fsync the parent directory to ensure the rename is durable.
if let Some(parent) = path.parent() {
- let dir = fs::File::open(parent)
+ let dir = compio::fs::File::open(parent)
+ .await
.note_descriptor_exhaustion(|| format!("opening directory {}", parent.display()))
.map_err(|e| SnapshotError::Persist {
stage: PersistStage::DirSync,
source: e,
})?;
- dir.sync_all().map_err(|e| SnapshotError::Persist {
+ dir.sync_all().await.map_err(|e| SnapshotError::Persist {
stage: PersistStage::DirSync,
source: e,
})?;
@@ -410,9 +427,10 @@ impl<M> SnapshotCoordinator<M> {
/// Create and durably persist a snapshot at `commit_op`, record the pairing, and
/// return its checksum. Does NOT drain the WAL: the caller must durably record
/// the pairing in the superblock first, so a crash between persist and drain
- /// recovers a consistent checkpoint with the WAL intact. Synchronous, since
- /// snapshot creation and `std::fs` persistence never await.
- fn persist_snapshot(
+ /// recovers a consistent checkpoint with the WAL intact. Snapshot creation and
+ /// encoding finish before the first await, preserving the captured frontier.
+ #[allow(clippy::future_not_send)]
+ async fn persist_snapshot(
&self,
stm: &M,
commit_op: u64,
@@ -430,7 +448,7 @@ impl<M> SnapshotCoordinator<M> {
let encoded = snapshot.encode()?;
let checksum = checkpoint_checksum(&encoded);
let path = self.snapshot_path();
- IggySnapshot::write_durably(&path, &encoded)?;
+ IggySnapshot::write_durably(&path, encoded).await?;
self.last_checkpoint.set((commit_op, checksum));
Ok(checksum)
}
@@ -827,7 +845,7 @@ pub struct IggyMetadata<C, J, S, M, SB = PingPongSuperblock> {
/// Scoped to the write itself, NOT to a whole checkpoint. A pending view persist
/// blocks every gated send behind it, including the ack path's
/// `send_prepare_ok`, so it must not also wait out a checkpoint's snapshot
- /// encode, two `std::fs` fsyncs and an async WAL drain. Whoever writes builds
+ /// encode, filesystem syncs and the WAL drain. Whoever writes builds
/// its `VsrState` inside this section with no await in between, so the last
/// writer carries the freshest view and the durable view cannot regress.
///
@@ -1806,7 +1824,12 @@ where
///
/// # Errors
/// [`StateTransferUnavailable`] naming why this replica cannot serve.
- pub fn state_transfer_offer(&self) -> Result<Rc<StateTransferOffer>, StateTransferUnavailable> {
+ #[allow(clippy::future_not_send)]
+ pub async fn state_transfer_offer(
+ &self,
+ ) -> Result<Rc<StateTransferOffer>, StateTransferUnavailable> {
+ let _checkpoint = self.checkpoint_lock.acquire().await;
+ let _snapshot_io = SNAPSHOT_IO.lock().await;
let consensus = self
.consensus
.as_ref()
@@ -1818,12 +1841,20 @@ where
.coordinator
.as_ref()
.ok_or(StateTransferUnavailable::NoCoordinator)?;
+ if let Some(cached) = self.transfer_offer_cache.borrow().as_ref() {
+ return Ok(Rc::clone(cached));
+ }
let path = coordinator.snapshot_path();
- if !path.exists() {
- return Err(StateTransferUnavailable::NoSnapshot);
+ let sealed = compio::fs::read(&path).await.map_err(|source| {
+ if source.kind() == io::ErrorKind::NotFound {
+ StateTransferUnavailable::NoSnapshot
+ } else {
+ StateTransferUnavailable::SnapshotUnreadable(source.into())
+ }
+ })?;
+ if !is_caught_up_primary(consensus) {
+ return Err(StateTransferUnavailable::NotCaughtUpPrimary);
}
- let sealed = std::fs::read(&path)
- .map_err(|source| StateTransferUnavailable::SnapshotUnreadable(source.into()))?;
// Verifies the trailer and hands back the payload alone.
let (payload, _) =
split_trailer(&sealed).map_err(StateTransferUnavailable::SnapshotUnreadable)?;
@@ -1835,16 +1866,6 @@ where
.map_err(StateTransferUnavailable::SnapshotUnreadable)?
.sequence_number();
- // Reuse the cached offer for this generation. Only the SNAPSHOT half is
- // expensive to rebuild, and the cached table is merely older, never
- // incoherent: its frontier is stamped at its own encode, and the receiver
- // replays everything above that frontier during tail repair.
- if let Some(cached) = self.transfer_offer_cache.borrow().as_ref()
- && cached.snapshot_seq == snapshot_seq
- {
- return Ok(Rc::clone(cached));
- }
-
let commit_op = consensus.commit_min();
let table = self.client_table.borrow().encode();
let offer = Rc::new(StateTransferOffer {
@@ -1985,8 +2006,7 @@ where
// withholds acks, but "should not be committing" is not an invariant
// this path can rest on.
//
- // Deadlock-free: nothing between here and the superblock write awaits,
- // and `write_superblock` takes no lock of its own.
+ // `write_superblock` takes no lock of its own.
let _install_gates = if snapshot_ahead {
let checkpoint = self.checkpoint_lock.acquire().await;
let superblock = if self.superblock.is_some() {
@@ -2007,8 +2027,9 @@ where
// snapshot.bin, regress its pairing, and rewind the STM below the
// applied frontier -- then panic on `set_commit_floor`'s anti-rewind
// assert with the damage already durable.
- let local_applied = consensus.commit_min();
- let snapshot_ahead = snapshot_seq > local_applied;
+ let mut local_applied = consensus.commit_min();
+ let mut snapshot_ahead = snapshot_seq > local_applied;
+ let persist_pairing = snapshot_ahead;
let snapshot_table = if snapshot_ahead {
let table = snapshot
@@ -2020,24 +2041,6 @@ where
} else {
None
};
- let protection_frontier = table_frontier.max(self.client_table_frontier.get());
- let replay_table = if snapshot_ahead && protection_frontier > snapshot_seq {
- snapshot_table
- } else if !snapshot_ahead
- && table_frontier > local_applied
- && self.replay_client_table.borrow().is_none()
- {
- let local_table = self.client_table.borrow();
- Some(if local_table.capacity_committed() {
- ClientTable::from_snapshot(local_table.to_snapshot())
- .map_err(SnapshotError::ClientTable)?
- } else {
- ClientTable::new(local_table.capacity())
- })
- } else {
- None
- };
-
if snapshot_ahead && let Some(journal) = &self.journal {
// Discard the WAL suffix above the incoming floor BEFORE anything
// installs: the commit walk matches entries by op number alone, so
@@ -2096,7 +2099,9 @@ where
coordinator.last_checkpoint().0
);
let checksum = checkpoint_checksum(snapshot_bytes);
- IggySnapshot::write_durably(&coordinator.snapshot_path(), snapshot_bytes)?;
+ self.clear_state_transfer_offer_cache();
+ IggySnapshot::write_durably(&coordinator.snapshot_path(), snapshot_bytes.to_vec())
+ .await?;
coordinator.seed_last_checkpoint(snapshot_seq, checksum);
tracing::info!(
checkpoint_op = snapshot_seq,
@@ -2110,8 +2115,14 @@ where
);
}
- self.mux_stm
- .restore_snapshot_in_place(snapshot.snapshot())?;
+ // Commits may progress while the snapshot reaches disk. Its bytes remain
+ // a valid checkpoint, but must not replace a newer in-memory state.
+ local_applied = consensus.commit_min();
+ snapshot_ahead = snapshot_seq > local_applied;
+ if snapshot_ahead {
+ self.mux_stm
+ .restore_snapshot_in_place(snapshot.snapshot())?;
+ }
} else {
tracing::info!(
snapshot_seq,
@@ -2121,6 +2132,23 @@ where
);
}
+ let protection_frontier = table_frontier.max(self.client_table_frontier.get());
+ let replay_table = if snapshot_ahead && protection_frontier > snapshot_seq {
+ snapshot_table
+ } else if !snapshot_ahead
+ && table_frontier > local_applied
+ && self.replay_client_table.borrow().is_none()
+ {
+ let local_table = self.client_table.borrow();
+ Some(if local_table.capacity_committed() {
+ ClientTable::from_snapshot(local_table.to_snapshot())
+ .map_err(SnapshotError::ClientTable)?
+ } else {
+ ClientTable::new(local_table.capacity())
+ })
+ } else {
+ None
+ };
if snapshot_ahead || replay_table.is_some() {
*self.replay_client_table.borrow_mut() = replay_table.map(RefCell::new);
}
@@ -2184,7 +2212,7 @@ where
// `verify_checkpoint_pairing` accepts -- so it is reported as a DEGRADED
// install rather than a failed one.
let mut pairing_durable = true;
- if snapshot_ahead && let Some(superblock) = self.superblock.as_ref() {
+ if persist_pairing && let Some(superblock) = self.superblock.as_ref() {
// Already under `_install_gates`, acquired above; re-acquiring here
// would deadlock on the same non-reentrant gate.
pairing_durable = self.write_superblock(consensus, superblock.as_ref()).await;
@@ -3623,12 +3651,11 @@ where
// then-current commit_min. The prepare being replicated appends regardless
// (see the phantom-op comment at the call site).
let client_table = self.client_table.borrow().to_snapshot();
- let checksum = match coordinator.persist_snapshot(
- &*self.mux_stm,
- snap_op,
- created_at,
- Some(client_table),
- ) {
+ self.clear_state_transfer_offer_cache();
+ let checksum = match coordinator
+ .persist_snapshot(&*self.mux_stm, snap_op, created_at, Some(client_table))
+ .await
+ {
Ok(checksum) => checksum,
Err(e) => {
error!(
@@ -4531,6 +4558,158 @@ mod tests {
use std::cell::RefCell;
use std::rc::Rc;
+ #[test]
+ fn given_cancelled_snapshot_writer_when_replaced_should_keep_the_new_snapshot() {
+ let runtime = server_common::executor::create_shard_executor().unwrap();
+ runtime.block_on(async {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("snapshot.bin");
+ let first = IggySnapshot::create(&TestMux::default(), 1, 1).unwrap();
+ let mut cancelled = Box::pin(first.persist(&path));
+ assert!(futures::poll!(&mut cancelled).is_pending());
+ drop(cancelled);
+
+ let replacement = IggySnapshot::create(&TestMux::default(), 2, 2).unwrap();
+ replacement.persist(&path).await.unwrap();
+ let (loaded, checksum) = IggySnapshot::load(&path).unwrap();
+ assert_eq!(loaded.sequence_number(), 2);
+ assert_eq!(
+ checksum,
+ checkpoint_checksum(&replacement.encode().unwrap())
+ );
+ assert!(!path.with_extension("bin.tmp").exists());
+ });
+ }
+
+ #[compio::test]
+ async fn given_cached_offer_when_installing_snapshot_should_invalidate_without_rereading() {
+ let directory = tempfile::tempdir().unwrap();
+ std::fs::create_dir(directory.path().join(crate::impls::METADATA_DIR)).unwrap();
+ let mut metadata = metadata_plane();
+ metadata.coordinator = Some(SnapshotCoordinator::new(
+ directory.path().to_path_buf(),
+ IggySnapshot::create,
+ ));
+ let path = metadata.coordinator.as_ref().unwrap().snapshot_path();
+ IggySnapshot::create(&TestMux::default(), 0, 1)
+ .unwrap()
+ .persist(&path)
+ .await
+ .unwrap();
+ let first = metadata.state_transfer_offer().await.unwrap();
+ std::fs::remove_file(&path).unwrap();
+ let cached = metadata.state_transfer_offer().await.unwrap();
+ assert!(
+ Rc::ptr_eq(&first, &cached),
+ "a cache hit must need no disk read"
+ );
+
+ let mut snapshot = IggySnapshot::create(&TestMux::default(), 1, 2).unwrap();
+ snapshot.snapshot_mut().client_table = Some(metadata.client_table.borrow().to_snapshot());
+ let incoming = snapshot.encode().unwrap();
+ metadata
+ .install_state_transfer(&incoming, ClientTable::new(CLIENTS_TABLE_MAX), 1, 1)
+ .await
+ .unwrap();
+ let replaced = metadata.state_transfer_offer().await.unwrap();
+ assert_eq!(replaced.snapshot_seq, 1);
+ assert!(!Rc::ptr_eq(&first, &replaced));
+
+ metadata.clear_state_transfer_offer_cache();
+ let mut damaged = std::fs::read(&path).unwrap();
+ damaged[0] ^= 1;
+ std::fs::write(&path, damaged).unwrap();
+ assert!(matches!(
+ metadata.state_transfer_offer().await,
+ Err(StateTransferUnavailable::SnapshotUnreadable(_))
+ ));
+ }
+
+ #[compio::test]
+ async fn given_commits_during_snapshot_io_when_installing_should_preserve_the_newer_frontier() {
+ const SNAPSHOT_OP: u64 = 1;
+ const LOCAL_OP: u64 = 2;
+ const TRANSFER_OP: u64 = 3;
+ const CLIENT: u128 = 9;
+ const USER: u32 = 7;
+ const BIND_VERIFIER: [u8; 32] = [0x5a; 32];
+ for table_frontier in [SNAPSHOT_OP, TRANSFER_OP] {
+ let directory = tempfile::tempdir().unwrap();
+ std::fs::create_dir(directory.path().join(crate::impls::METADATA_DIR)).unwrap();
+ let mut metadata = metadata_plane();
+ metadata.coordinator = Some(SnapshotCoordinator::new(
+ directory.path().to_path_buf(),
+ IggySnapshot::create,
+ ));
+ let mut table = ClientTable::new(CLIENTS_TABLE_MAX);
+ table.commit_capacity(CLIENTS_TABLE_MAX).unwrap();
+ let mut snapshot = IggySnapshot::create(&TestMux::default(), SNAPSHOT_OP, 1).unwrap();
+ snapshot.snapshot_mut().client_table = Some(table.to_snapshot());
+ let incoming = snapshot.encode().unwrap();
+ let held = SNAPSHOT_IO.lock().await;
+ let mut install = Box::pin(metadata.install_state_transfer(
+ &incoming,
+ table,
+ table_frontier,
+ table_frontier,
+ ));
+ assert!(futures::poll!(&mut install).is_pending());
+ let consensus = metadata.consensus.as_ref().unwrap();
+ consensus.advance_commit_max(LOCAL_OP);
+ consensus.set_commit_floor(LOCAL_OP);
+ {
+ let mut local_table = metadata.client_table.borrow_mut();
+ local_table.commit_capacity(CLIENTS_TABLE_MAX).unwrap();
+ local_table
+ .commit_register(
+ CLIENT,
+ USER,
+ BIND_VERIFIER,
+ register_reply(CLIENT, LOCAL_OP),
+ )
+ .unwrap();
+ }
+ drop(held);
+ let outcome = install.await.unwrap();
+ assert_eq!(outcome.installed_frontier, LOCAL_OP);
+ assert_eq!(consensus.commit_min(), LOCAL_OP);
+ if table_frontier > LOCAL_OP {
+ assert_eq!(metadata.client_table.borrow().get_epoch(CLIENT), None);
+ let replay = metadata.replay_client_table.borrow();
+ assert_eq!(
+ replay.as_ref().unwrap().borrow().get_epoch(CLIENT),
+ Some(LOCAL_OP),
+ "tail replay must retain the session committed during snapshot I/O"
+ );
+ } else {
+ assert_eq!(
+ metadata.client_table.borrow().get_epoch(CLIENT),
+ Some(LOCAL_OP)
+ );
+ assert!(metadata.replay_client_table.borrow().is_none());
+ }
+ }
+ }
+
+ #[compio::test]
+ async fn given_failed_snapshot_replacement_when_retrying_should_publish_valid_bytes() {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("snapshot.bin");
+ std::fs::create_dir(&path).unwrap();
+ let snapshot = IggySnapshot::create(&TestMux::default(), 1, 1).unwrap();
+ assert!(matches!(
+ snapshot.persist(&path).await,
+ Err(SnapshotError::Persist {
+ stage: PersistStage::Rename,
+ ..
+ })
+ ));
+ std::fs::remove_dir(&path).unwrap();
+ snapshot.persist(&path).await.unwrap();
+ let (loaded, _) = IggySnapshot::load(&path).unwrap();
+ assert_eq!(loaded.sequence_number(), 1);
+ }
+
#[test]
fn eviction_reason_splits_client_and_internal_ops() {
// Client-allowed op with a bad body evicts InvalidRequestBody; an internal
diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs
index ec2f2444e..cebed0189 100644
--- a/core/metadata/src/impls/recovery.rs
+++ b/core/metadata/src/impls/recovery.rs
@@ -1058,6 +1058,7 @@ mod tests {
let snapshot = protected_snapshot(42);
snapshot
.persist(&metadata_dir.join("snapshot.bin"))
+ .await
.unwrap();
let recovered = recover::<TestStm>(
@@ -1290,6 +1291,7 @@ mod tests {
let snapshot = protected_snapshot(5);
snapshot
.persist(&metadata_dir.join("snapshot.bin"))
+ .await
.unwrap();
// WAL has ops 1-10
@@ -1353,6 +1355,7 @@ mod tests {
for snapshot in [IggySnapshot::new(2), malformed] {
snapshot
.persist(&metadata_dir.join("snapshot.bin"))
+ .await
.unwrap();
let result = recover::<TestStm>(
dir.path(),
@@ -1417,6 +1420,7 @@ mod tests {
snapshot.snapshot_mut().client_table = Some(at_checkpoint.to_snapshot());
snapshot
.persist(&metadata_dir.join("snapshot.bin"))
+ .await
.unwrap();
{
@@ -1713,13 +1717,13 @@ mod tests {
}
}
- #[test]
- fn snapshot_persist_load_roundtrip() {
+ #[compio::test]
+ async fn snapshot_persist_load_roundtrip() {
let dir = tempdir().unwrap();
let path = dir.path().join("snapshot.bin");
let snapshot = protected_snapshot(99);
- snapshot.persist(&path).unwrap();
+ snapshot.persist(&path).await.unwrap();
let (loaded, checksum) = IggySnapshot::load(&path).unwrap();
assert_eq!(loaded.sequence_number(), 99);
@@ -1742,13 +1746,13 @@ mod tests {
));
}
- #[test]
- fn snapshot_refuses_absent_truncated_and_damaged_integrity_framing() {
+ #[compio::test]
+ async fn snapshot_refuses_absent_truncated_and_damaged_integrity_framing() {
let dir = tempdir().unwrap();
let path = dir.path().join("snapshot.bin");
let snapshot = protected_snapshot(99);
let payload = snapshot.encode().unwrap();
- snapshot.persist(&path).unwrap();
+ snapshot.persist(&path).await.unwrap();
let sealed = std::fs::read(&path).unwrap();
assert_eq!(
IggySnapshot::decode(&payload).unwrap().sequence_number(),
@@ -1940,8 +1944,10 @@ mod tests {
let dir = tempdir().unwrap();
let metadata_dir = dir.path().join("metadata");
std::fs::create_dir_all(&metadata_dir).unwrap();
- IggySnapshot::write_durably(&metadata_dir.join("snapshot.bin"), &foreign).unwrap();
let state = vsr_state_with_checkpoint(CHECKPOINT_OP, checkpoint_checksum(&foreign));
+ IggySnapshot::write_durably(&metadata_dir.join("snapshot.bin"), foreign)
+ .await
+ .unwrap();
{
let superblock = PingPongSuperblock::open(&metadata_dir).await.unwrap();
superblock.write(&state.to_bytes()).await.unwrap();
@@ -2005,8 +2011,11 @@ mod tests {
let dir = tempdir().unwrap();
let metadata_dir = dir.path().join("metadata");
std::fs::create_dir_all(&metadata_dir).unwrap();
- IggySnapshot::write_durably(&metadata_dir.join("snapshot.bin"), &on_disk).unwrap();
- let state = vsr_state_with_checkpoint(CHECKPOINT_OP, checkpoint_checksum(&on_disk));
+ let checksum = checkpoint_checksum(&on_disk);
+ IggySnapshot::write_durably(&metadata_dir.join("snapshot.bin"), on_disk)
+ .await
+ .unwrap();
+ let state = vsr_state_with_checkpoint(CHECKPOINT_OP, checksum);
{
let superblock = PingPongSuperblock::open(&metadata_dir).await.unwrap();
superblock.write(&state.to_bytes()).await.unwrap();
@@ -2024,8 +2033,7 @@ mod tests {
.unwrap();
assert_eq!(recovered.snapshot_checkpoint.0, CHECKPOINT_OP);
assert_eq!(
- recovered.snapshot_checkpoint.1,
- checkpoint_checksum(&on_disk),
+ recovered.snapshot_checkpoint.1, checksum,
"the verified pairing must be the checksum of the bytes on disk"
);
}
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs
index 2a21d1e5a..5ffabdfb5 100644
--- a/core/partitions/src/iggy_partition.rs
+++ b/core/partitions/src/iggy_partition.rs
@@ -1667,7 +1667,7 @@ where
let Some(persistence) = &self.persistence else {
return (Vec::new(), Vec::new());
};
- let (segments, offsets) = persistence.take_dirty_files();
+ let (segments, offsets, dirty_directories) = persistence.take_dirty_files();
let mut paths = Vec::with_capacity(
segments.len() * 2
+ offsets
@@ -1708,7 +1708,10 @@ where
}
}
let mut directories = Vec::with_capacity(ConsumerKind::COUNT + 2);
- for directory in self.consumer_offset_dirs.iter().flatten() {
+ for (directory, dirty) in self.consumer_offset_dirs.iter().zip(dirty_directories) {
+ let Some(directory) = directory.as_ref().filter(|_| dirty) else {
+ continue;
+ };
let path = std::path::PathBuf::from(directory);
directories.push(path.clone());
if let Some(parent) = path.parent()
@@ -14950,6 +14953,31 @@ mod tests {
assert!(directories.contains(&directory.path().join("offsets/consumers")));
assert!(directories.contains(&directory.path().join("offsets/groups")));
assert!(directories.contains(&directory.path().to_path_buf()));
+
+ let (_, clean) = partition.persistence_checkpoint_files(&repair_config());
+ assert_eq!(clean, [directory.path().to_path_buf()]);
+
+ let kind = ConsumerKind::ExternalGroup;
+ let external = directory.path().join("offsets/external_groups");
+ partition.consumer_offset_dirs[kind.index()] =
+ Some(external.to_string_lossy().into_owned());
+ let persistence = Rc::clone(partition.persistence.as_ref().unwrap());
+ persistence.mark_offset_dirty(kind.index(), 7, true);
+ persistence.mark_offset_dirty(kind.index(), 7, false);
+ let (files, deleted) = partition.persistence_checkpoint_files(&repair_config());
+ assert!(files.is_empty(), "the last key was deleted");
+ assert_eq!(
+ deleted,
+ [
+ external,
+ directory.path().join("offsets"),
+ directory.path().to_path_buf()
+ ]
+ );
+
+ persistence.reset(0, None);
+ let (_, reset) = partition.persistence_checkpoint_files(&repair_config());
+ assert_eq!(reset.len(), ConsumerKind::COUNT + 2);
}
async fn publish_empty_retry_checkpoint(
diff --git a/core/partitions/src/messages_writer.rs b/core/partitions/src/messages_writer.rs
index c836f94dd..7b1db97cc 100644
--- a/core/partitions/src/messages_writer.rs
+++ b/core/partitions/src/messages_writer.rs
@@ -64,7 +64,7 @@ impl MessagesWriter {
.map_err(|_| IggyError::CannotReadFile)?;
if let Some(preallocate_size) = preallocate_size {
- preallocate_file(&file, Path::new(file_path), preallocate_size.as_bytes_u64());
+ preallocate_file(&file, Path::new(file_path), preallocate_size.as_bytes_u64()).await;
}
if file_exists {
diff --git a/core/partitions/src/persistence.rs b/core/partitions/src/persistence.rs
index ef0964ef5..0e212d1df 100644
--- a/core/partitions/src/persistence.rs
+++ b/core/partitions/src/persistence.rs
@@ -147,6 +147,7 @@ pub struct PartitionPersistence<S: DurableStorage = DiskStorage> {
checkpoint_needed: Cell<bool>,
dirty_segments: RefCell<BTreeSet<u64>>,
dirty_offsets: [RefCell<BTreeSet<u32>>; ConsumerKind::COUNT],
+ dirty_offset_directories: [Cell<bool>; ConsumerKind::COUNT],
purge_generation: Cell<u64>,
purge_floor: Cell<u64>,
capacity: u64,
@@ -570,6 +571,8 @@ impl<S: DurableStorage> PartitionPersistence<S> {
checkpoint_needed: Cell::new(false),
dirty_segments: RefCell::new(BTreeSet::new()),
dirty_offsets: std::array::from_fn(|_| RefCell::new(BTreeSet::new())),
+ // Recovery may have completed a rename or unlink without its final barrier.
+ dirty_offset_directories: std::array::from_fn(|_| Cell::new(true)),
purge_generation: Cell::new(journal.purge_marker().0),
purge_floor: Cell::new(journal.purge_marker().1),
capacity,
@@ -857,6 +860,7 @@ impl<S: DurableStorage> PartitionPersistence<S> {
}
pub fn mark_offset_dirty(&self, kind_index: usize, consumer_id: u32, exists: bool) {
+ self.dirty_offset_directories[kind_index].set(true);
let mut offsets = self.dirty_offsets[kind_index].borrow_mut();
if exists {
offsets.insert(consumer_id);
@@ -869,12 +873,19 @@ impl<S: DurableStorage> PartitionPersistence<S> {
self.offset_files.retire_all();
}
- pub fn take_dirty_files(&self) -> (BTreeSet<u64>, [BTreeSet<u32>; ConsumerKind::COUNT]) {
+ pub fn take_dirty_files(
+ &self,
+ ) -> (
+ BTreeSet<u64>,
+ [BTreeSet<u32>; ConsumerKind::COUNT],
+ [bool; ConsumerKind::COUNT],
+ ) {
(
std::mem::take(&mut *self.dirty_segments.borrow_mut()),
std::array::from_fn(|index| {
std::mem::take(&mut *self.dirty_offsets[index].borrow_mut())
}),
+ std::array::from_fn(|index| self.dirty_offset_directories[index].replace(false)),
)
}
@@ -919,6 +930,9 @@ impl<S: DurableStorage> PartitionPersistence<S> {
return;
}
self.checkpoint_requested.set(self.checkpoint.get());
+ for dirty in &self.dirty_offset_directories {
+ dirty.set(true);
+ }
self.certified_log_view.set(None);
self.requested_log_view.set(None);
let epoch = self.epoch.get().wrapping_add(1);
@@ -990,6 +1004,9 @@ impl<S: DurableStorage> PartitionPersistence<S> {
self.certified_log_view.set(None);
self.requested_log_view.set(None);
self.checkpoint_requested.set(op);
+ for dirty in &self.dirty_offset_directories {
+ dirty.set(true);
+ }
let epoch = self.epoch.get().wrapping_add(1);
self.epoch.set(epoch);
self.queue.borrow_mut().clear();
@@ -1691,13 +1708,15 @@ mod tests {
persistence.mark_offset_dirty(0, 7, true);
persistence.mark_offset_dirty(0, 7, false);
persistence.mark_offset_dirty(1, 9, true);
- let (segments, offsets) = persistence.take_dirty_files();
+ let (segments, offsets, directories) = persistence.take_dirty_files();
assert_eq!(segments.into_iter().collect::<Vec<_>>(), vec![0]);
assert!(offsets[0].is_empty());
assert!(offsets[1].contains(&9));
- let (segments, offsets) = persistence.take_dirty_files();
+ assert_eq!(directories, [true; ConsumerKind::COUNT]);
+ let (segments, offsets, directories) = persistence.take_dirty_files();
assert!(segments.is_empty());
assert!(offsets.iter().all(BTreeSet::is_empty));
+ assert_eq!(directories, [false; ConsumerKind::COUNT]);
let first = prepare(1, 0);
let checkpoint_header = *first.header();
let second = prepare(2, first.header().checksum);
diff --git a/core/partitions/src/segment_recovery.rs b/core/partitions/src/segment_recovery.rs
index 9375ea079..662b9c602 100644
--- a/core/partitions/src/segment_recovery.rs
+++ b/core/partitions/src/segment_recovery.rs
@@ -30,16 +30,18 @@
use crate::iggy_index::IGGY_INDEX_SIZE;
use crate::segment_anchor::ANCHOR_EXTENSION;
-use crate::state_transfer::STAGING_SUFFIX;
+use crate::state_transfer::{STAGING_SUFFIX, segment_dir_entries};
use crate::{IggyIndex, IggyIndexReader, PartitionsConfig, Segment};
+use compio::buf::{IntoInner, IoBuf};
+use compio::fs;
+use compio::io::{AsyncReadAtExt, AsyncWriteAtExt};
use iggy_common::{IggyByteSize, IggyError, MAX_MESSAGE_SIZE_UPPER_BYTES, PartitionStats};
use server_common::fatal::NoteDescriptorExhaustion;
+use server_common::fs_utils::truncate_file;
use server_common::send_messages::{BatchHeader, COMMAND_HEADER_SIZE, decode_batch_slice};
use server_common::sharding::IggyNamespace;
use server_common::{SegmentStorage, yield_to_reactor};
-use std::fs;
use std::io;
-use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};
use tracing::{error, info, warn};
@@ -613,7 +615,7 @@ pub async fn load_persisted_segments_with_checkpoint(
// collects ARE the post-sweep start-offset set. Note the error policy is
// the collect side's (NotFound => empty, anything else => refuse boot); the
// sweep's silent return would swallow an EACCES that must not be ignored.
- let mut start_offsets = sweep_scratch_files_and_collect_offsets(&partition_path)?;
+ let mut start_offsets = sweep_scratch_files_and_collect_offsets(&partition_path).await?;
start_offsets.sort_unstable();
if let Some(checkpoint) = checkpoint {
start_offsets.retain(|offset| {
@@ -636,7 +638,7 @@ pub async fn load_persisted_segments_with_checkpoint(
config.get_messages_path(stream_id, topic_id, partition_id, start_offset);
let index_path = config.get_index_path(stream_id, topic_id, partition_id, start_offset);
- let raw_messages_size = file_len(&messages_path)?;
+ let raw_messages_size = file_len(&messages_path).await?;
let checkpoint_segment =
checkpoint.is_some_and(|checkpoint| checkpoint.start_offset == start_offset);
let messages_size = checkpoint
@@ -652,7 +654,7 @@ pub async fn load_persisted_segments_with_checkpoint(
);
}
let bounds = if checkpoint_segment {
- let messages = open_messages_file(identity, &messages_path)?;
+ let messages = open_messages_file(identity, &messages_path).await?;
let mut scanner = FileScanner::new(&messages, messages_size, &mut scratch);
recover_by_walking_log(
identity,
@@ -723,8 +725,8 @@ pub async fn load_persisted_segments_with_checkpoint(
// Staged now so pass C can install it with one atomic rename, and so
// a long chain never holds more than one rebuilt index in memory.
- let rebuilt_index_staging = match &bounds.rebuilt_index {
- Some(entries) => Some(stage_rebuilt_index(&index_path, entries)?),
+ let rebuilt_index_staging = match bounds.rebuilt_index {
+ Some(entries) => Some(stage_rebuilt_index(&index_path, entries).await?),
None => None,
};
@@ -769,7 +771,8 @@ pub async fn load_persisted_segments_with_checkpoint(
&plan.messages_path,
&plan.index_path,
plan.segment.start_offset,
- )?;
+ )
+ .await?;
}
// Log first, index second: an index is kept only when a whole batch
// verifies at its LAST entry's position, so the walked log length
@@ -785,12 +788,12 @@ pub async fn load_persisted_segments_with_checkpoint(
// pair converges.
if checkpoint.is_none_or(|checkpoint| checkpoint.start_offset != plan.segment.start_offset)
{
- truncate_to(&plan.messages_path, messages_size)?;
+ truncate_to(&plan.messages_path, messages_size).await?;
}
if let Some(staging_path) = &plan.rebuilt_index_staging {
- install_rebuilt_index(staging_path, &plan.index_path, identity.partition_path)?;
+ install_rebuilt_index(staging_path, &plan.index_path, identity.partition_path).await?;
} else {
- truncate_to(&plan.index_path, plan.index_size)?;
+ truncate_to(&plan.index_path, plan.index_size).await?;
}
let mut storage = if checkpoint.is_some() {
@@ -1135,12 +1138,13 @@ async fn ensure_contiguous_chain(
/// Sweeps boot-time scratch (`.staging` spill, orphan `.index`) and returns the
/// start offset parsed out of every remaining zero-padded `.log` file name. A
/// missing directory means a never-persisted partition.
-fn sweep_scratch_files_and_collect_offsets(
+async fn sweep_scratch_files_and_collect_offsets(
partition_path: &str,
) -> Result<Vec<u64>, PartitionRecoveryError> {
// The recovery opens are read-only, but a failed one fails the partition
// load, which stops the server at boot, so they count like write opens.
- let entries = match fs::read_dir(partition_path)
+ let entries = match segment_dir_entries(partition_path)
+ .await
.note_descriptor_exhaustion(|| format!("listing directory {partition_path}"))
{
Ok(entries) => entries,
@@ -1158,8 +1162,7 @@ fn sweep_scratch_files_and_collect_offsets(
let mut orphan_candidates = Vec::new();
let mut log_stems = std::collections::HashSet::new();
let mut start_offsets = Vec::new();
- for entry in entries.flatten() {
- let path = entry.path();
+ for path in entries {
let Some(as_str) = path.to_str() else {
continue;
};
@@ -1191,7 +1194,7 @@ fn sweep_scratch_files_and_collect_offsets(
.is_some_and(|stem| log_stems.contains(stem))
}));
for path in swept {
- if let Err(error) = fs::remove_file(&path) {
+ if let Err(error) = fs::remove_file(&path).await {
warn!(
partition_path,
path = %path.display(),
@@ -1211,8 +1214,8 @@ fn sweep_scratch_files_and_collect_offsets(
/// `EIO` into 0 would route a healthy segment into recover-as-empty, fencing
/// it out of service (worst route: an index stat error floors a healthy
/// sealed index to a 0-byte target while its entries still load).
-fn file_len(path: &str) -> Result<u64, PartitionRecoveryError> {
- match fs::metadata(path) {
+async fn file_len(path: &str) -> Result<u64, PartitionRecoveryError> {
+ match fs::metadata(path).await {
Ok(metadata) => Ok(metadata.len()),
Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(0),
Err(source) => {
@@ -1242,12 +1245,11 @@ fn file_len(path: &str) -> Result<u64, PartitionRecoveryError> {
///
/// Stats the file fresh instead of trusting a length carried from pass A: the
/// whole chain was walked in between, and the mutation must key on what is on
-/// disk now. Synchronous `std::fs` on purpose (see [`FileScanner`]). The
-/// fsync bounds the crash window: a power cut right after `set_len` may
+/// disk now. The fsync bounds the crash window: a power cut right after `set_len` may
/// re-present the torn tail on the next boot, which only walks and truncates
/// again (idempotent), but the sync keeps the common case deterministic.
-fn truncate_to(path: &str, target_size: u64) -> Result<(), PartitionRecoveryError> {
- let current_size = file_len(path)?;
+async fn truncate_to(path: &str, target_size: u64) -> Result<(), PartitionRecoveryError> {
+ let current_size = file_len(path).await?;
if current_size == target_size {
return Ok(());
}
@@ -1275,6 +1277,7 @@ fn truncate_to(path: &str, target_size: u64) -> Result<(), PartitionRecoveryErro
let file = fs::OpenOptions::new()
.write(true)
.open(path)
+ .await
.note_descriptor_exhaustion(|| format!("opening {path}"))
.map_err(|source| {
error!(
@@ -1284,7 +1287,7 @@ fn truncate_to(path: &str, target_size: u64) -> Result<(), PartitionRecoveryErro
);
PartitionRecoveryError::from(IggyError::CannotWriteToFile)
})?;
- file.set_len(target_size).map_err(|source| {
+ truncate_file(&file, target_size).await.map_err(|source| {
error!(
path,
target_size,
@@ -1293,7 +1296,7 @@ fn truncate_to(path: &str, target_size: u64) -> Result<(), PartitionRecoveryErro
);
PartitionRecoveryError::from(IggyError::CannotWriteToFile)
})?;
- file.sync_all().map_err(|source| {
+ file.sync_all().await.map_err(|source| {
error!(
path,
error = %source,
@@ -1316,13 +1319,17 @@ fn truncate_to(path: &str, target_size: u64) -> Result<(), PartitionRecoveryErro
/// The staging file is pure scratch until pass C renames it into place: the
/// boot sweep unlinks orphaned `*.staging` files, so a crash anywhere before
/// the rename costs nothing.
-fn stage_rebuilt_index(index_path: &str, entries: &[u8]) -> Result<String, PartitionRecoveryError> {
+async fn stage_rebuilt_index(
+ index_path: &str,
+ entries: Vec<u8>,
+) -> Result<String, PartitionRecoveryError> {
let staging_path = format!("{index_path}{STAGING_SUFFIX}");
- let file = fs::OpenOptions::new()
+ let mut file = fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&staging_path)
+ .await
.note_descriptor_exhaustion(|| format!("opening {staging_path}"))
.map_err(|source| {
error!(
@@ -1332,7 +1339,7 @@ fn stage_rebuilt_index(index_path: &str, entries: &[u8]) -> Result<String, Parti
);
PartitionRecoveryError::from(IggyError::CannotWriteToFile)
})?;
- file.write_all_at(entries, 0).map_err(|source| {
+ file.write_all_at(entries, 0).await.0.map_err(|source| {
error!(
path = %staging_path,
error = %source,
@@ -1340,7 +1347,7 @@ fn stage_rebuilt_index(index_path: &str, entries: &[u8]) -> Result<String, Parti
);
PartitionRecoveryError::from(IggyError::CannotWriteToFile)
})?;
- file.sync_all().map_err(|source| {
+ file.sync_all().await.map_err(|source| {
error!(
path = %staging_path,
error = %source,
@@ -1355,37 +1362,41 @@ fn stage_rebuilt_index(index_path: &str, entries: &[u8]) -> Result<String, Parti
/// atomic commit point: the on-disk index is either the old one holding no
/// whole entry (whose walk re-runs the rebuild) or the complete rebuilt one,
/// never a mix of pages from both.
-fn install_rebuilt_index(
+async fn install_rebuilt_index(
staging_path: &str,
index_path: &str,
partition_path: &str,
) -> Result<(), PartitionRecoveryError> {
- fs::rename(staging_path, index_path).map_err(|source| {
- error!(
- from = %staging_path,
- to = %index_path,
- error = %source,
- "failed to rename a rebuilt sparse index into place during recovery"
- );
- PartitionRecoveryError::from(IggyError::CannotWriteToFile)
- })?;
- fsync_dir(partition_path)
-}
-
-/// Makes renames and new files in `dir` durable. Synchronous like every
-/// other mutation in this module (see [`FileScanner`]).
-fn fsync_dir(dir: &str) -> Result<(), PartitionRecoveryError> {
- fs::File::open(dir)
- .note_descriptor_exhaustion(|| format!("opening directory {dir}"))
- .and_then(|handle| handle.sync_all())
+ fs::rename(staging_path, index_path)
+ .await
.map_err(|source| {
error!(
- dir,
+ from = %staging_path,
+ to = %index_path,
error = %source,
- "failed to fsync a directory during recovery"
+ "failed to rename a rebuilt sparse index into place during recovery"
);
- PartitionRecoveryError::from(IggyError::CannotSyncFile)
- })
+ PartitionRecoveryError::from(IggyError::CannotWriteToFile)
+ })?;
+ fsync_dir(partition_path).await
+}
+
+/// Makes renames and new files in `dir` durable.
+async fn fsync_dir(dir: &str) -> Result<(), PartitionRecoveryError> {
+ let synced = async {
+ let handle = fs::File::open(dir)
+ .await
+ .note_descriptor_exhaustion(|| format!("opening directory {dir}"))?;
+ handle.sync_all().await
+ };
+ synced.await.map_err(|source| {
+ error!(
+ dir,
+ error = %source,
+ "failed to fsync a directory during recovery"
+ );
+ PartitionRecoveryError::from(IggyError::CannotSyncFile)
+ })
}
/// Moves a segment pair that recovery proved unreadable into a fresh
@@ -1399,14 +1410,14 @@ fn fsync_dir(dir: &str) -> Result<(), PartitionRecoveryError> {
/// sweeps orphaned indexes, so a crash between the two creates leaves only
/// states a later boot already understands (segment absent, or one orphan
/// index).
-fn fence_unrecoverable_segment_files(
+async fn fence_unrecoverable_segment_files(
identity: PartitionIdentity<'_>,
messages_path: &str,
index_path: &str,
start_offset: u64,
) -> Result<(), PartitionRecoveryError> {
- let log_bytes = file_len(messages_path)?;
- let index_bytes = file_len(index_path)?;
+ let log_bytes = file_len(messages_path).await?;
+ let index_bytes = file_len(index_path).await?;
if log_bytes == 0 && index_bytes == 0 {
return Ok(());
}
@@ -1416,7 +1427,7 @@ fn fence_unrecoverable_segment_files(
// `create_dir`, not `create_dir_all`: success is the claim on this
// suffix, and merging into an existing fence would mix evidence from
// two incidents.
- match fs::create_dir(&candidate) {
+ match fs::create_dir(&candidate).await {
Ok(()) => {
fenced_dir = Some(candidate);
break;
@@ -1441,19 +1452,19 @@ fn fence_unrecoverable_segment_files(
};
let fenced_log = fenced_target(&fenced_dir, messages_path)?;
let fenced_index = fenced_target(&fenced_dir, index_path)?;
- rename_into_fence(messages_path, &fenced_log)?;
- rename_into_fence(index_path, &fenced_index)?;
- seed_empty_file(index_path)?;
- seed_empty_file(messages_path)?;
+ rename_into_fence(messages_path, &fenced_log).await?;
+ rename_into_fence(index_path, &fenced_index).await?;
+ seed_empty_file(index_path).await?;
+ seed_empty_file(messages_path).await?;
// The fence directory's new dirents, the partition directory's renames
// plus fresh files, and the parent's new fence-directory dirent.
- fsync_dir(&fenced_dir)?;
- fsync_dir(identity.partition_path)?;
+ fsync_dir(&fenced_dir).await?;
+ fsync_dir(identity.partition_path).await?;
if let Some(parent) = Path::new(identity.partition_path)
.parent()
.and_then(Path::to_str)
{
- fsync_dir(parent)?;
+ fsync_dir(parent).await?;
}
warn!(
stream_id = identity.stream_id,
@@ -1486,8 +1497,8 @@ fn fenced_target(fenced_dir: &str, source_path: &str) -> Result<PathBuf, Partiti
)
}
-fn rename_into_fence(source_path: &str, target: &Path) -> Result<(), PartitionRecoveryError> {
- match fs::rename(source_path, target) {
+async fn rename_into_fence(source_path: &str, target: &Path) -> Result<(), PartitionRecoveryError> {
+ match fs::rename(source_path, target).await {
Ok(()) => Ok(()),
// A missing index beside a present log has nothing to move.
Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()),
@@ -1503,18 +1514,21 @@ fn rename_into_fence(source_path: &str, target: &Path) -> Result<(), PartitionRe
}
}
-fn seed_empty_file(path: &str) -> Result<(), PartitionRecoveryError> {
- fs::File::create(path)
- .note_descriptor_exhaustion(|| format!("creating {path}"))
- .and_then(|file| file.sync_all())
- .map_err(|source| {
- error!(
- path,
- error = %source,
- "failed to seed a fresh empty segment file after fencing"
- );
- PartitionRecoveryError::from(IggyError::CannotWriteToFile)
- })
+async fn seed_empty_file(path: &str) -> Result<(), PartitionRecoveryError> {
+ let seeded = async {
+ let file = fs::File::create(path)
+ .await
+ .note_descriptor_exhaustion(|| format!("creating {path}"))?;
+ file.sync_all().await
+ };
+ seeded.await.map_err(|source| {
+ error!(
+ path,
+ error = %source,
+ "failed to seed a fresh empty segment file after fencing"
+ );
+ PartitionRecoveryError::from(IggyError::CannotWriteToFile)
+ })
}
/// Index anchors for one segment: `(entry_count, first, last)`.
@@ -1533,7 +1547,7 @@ async fn load_index_anchors(
identity: PartitionIdentity<'_>,
index_path: &str,
) -> Result<(u64, Option<IggyIndex>, Option<IggyIndex>), PartitionRecoveryError> {
- if file_len(index_path)? == 0 {
+ if file_len(index_path).await? == 0 {
return Ok((0, None, None));
}
// Opened here, not by `IggyIndexReader::new`, which drops the `io::Error`.
@@ -1639,7 +1653,7 @@ async fn recover_segment_bounds(
)
.await?;
- let messages = open_messages_file(identity, messages_path)?;
+ let messages = open_messages_file(identity, messages_path).await?;
let mut scanner = FileScanner::new(&messages, messages_size, scratch);
if !validation.structurally_consistent {
return recover_by_walking_log(
@@ -1845,7 +1859,7 @@ async fn recover_segment_bounds(
// path performs, and rebuilds the index from the batches it proves so
// a sealed segment does not pay a full-scan poll penalty forever.
_ if messages_size > 0 => {
- let messages = open_messages_file(identity, messages_path)?;
+ let messages = open_messages_file(identity, messages_path).await?;
let mut scanner = FileScanner::new(&messages, messages_size, scratch);
recover_by_walking_log(
identity,
@@ -1893,6 +1907,7 @@ fn ensure_fsynced_rebuild_reaches(
/// entries the log cannot back. Every batch is checksum-verified, as in the
/// anchored walk; here the FILENAME is the only anchor at all, and the
/// header decode checks a length, not a checksum.
+#[allow(clippy::too_many_lines)]
async fn recover_by_walking_log(
identity: PartitionIdentity<'_>,
scanner: &mut FileScanner<'_>,
@@ -1908,7 +1923,7 @@ async fn recover_by_walking_log(
let mut rebuilt_index = Vec::new();
let mut last_indexed_position: Option<u64> = None;
while position < messages_size {
- let Some(header) = header_at(identity, scanner, messages_path, position)? else {
+ let Some(header) = header_at(identity, scanner, messages_path, position).await? else {
break;
};
let extent = position.saturating_add(header.total_size() as u64);
@@ -1921,7 +1936,8 @@ async fn recover_by_walking_log(
messages_path,
position,
header.total_size(),
- )?;
+ )
+ .await?;
if !verifies {
break;
}
@@ -2042,7 +2058,7 @@ async fn walk_chain_from_anchor(
// already proved makes the expectation the LOG's own.
let mut expectation_from_log = false;
while position < messages_size {
- let Some(header) = header_at(identity, scanner, messages_path, position)? else {
+ let Some(header) = header_at(identity, scanner, messages_path, position).await? else {
break;
};
let extent = position.saturating_add(header.total_size() as u64);
@@ -2063,7 +2079,9 @@ async fn walk_chain_from_anchor(
messages_path,
position,
header.total_size(),
- )? {
+ )
+ .await?
+ {
break;
}
if header.partition_id != identity.partition_id as u64 {
@@ -2184,7 +2202,7 @@ impl<'scan> IndexLogScanner<'scan> {
}
}
- fn entry_matches(
+ async fn entry_matches(
&mut self,
offset: u64,
timestamp: u64,
@@ -2201,8 +2219,8 @@ impl<'scan> IndexLogScanner<'scan> {
let fill = usize::try_from((self.file_len - position).min(SCAN_WINDOW_CAPACITY as u64))
.unwrap_or(SCAN_WINDOW_CAPACITY);
self.window.resize(fill, 0);
- self.file
- .read_exact_at(&mut self.window[..], position)
+ read_scan_window(self.file, self.window, position)
+ .await
.map_err(|source| {
error!(
stream_id = self.identity.stream_id,
@@ -2265,6 +2283,7 @@ impl<'scan> IndexLogScanner<'scan> {
/// every [`INDEX_SCAN_YIELD_STRIDE`] entries: an entry that overshoots the
/// log reads nothing from it, so there is no refill to key the yield on, and
/// an index that outran a truncated log is full of exactly those.
+#[allow(clippy::too_many_lines)]
async fn find_provable_index_anchor(
identity: PartitionIdentity<'_>,
index_path: &str,
@@ -2283,6 +2302,7 @@ async fn find_provable_index_anchor(
});
};
let file = fs::File::open(index_path)
+ .await
.note_descriptor_exhaustion(|| format!("opening {index_path}"))
.map_err(|source| {
error!(
@@ -2301,18 +2321,22 @@ async fn find_provable_index_anchor(
let mut budgeted_down_to = messages_size;
let mut searched_entries = 0u64;
loop {
- file.read_exact_at(&mut raw, entry_index * IGGY_INDEX_SIZE as u64)
- .map_err(|source| {
- error!(
- stream_id = identity.stream_id,
- topic_id = identity.topic_id,
- partition_id = identity.partition_id,
- path = %index_path,
- error = %source,
- "failed to read a sparse index entry for anchor search during recovery"
- );
- PartitionRecoveryError::from(IggyError::CannotReadFile)
- })?;
+ let (read, bytes) = file
+ .read_exact_at(raw, entry_index * IGGY_INDEX_SIZE as u64)
+ .await
+ .into();
+ raw = bytes;
+ read.map_err(|source| {
+ error!(
+ stream_id = identity.stream_id,
+ topic_id = identity.topic_id,
+ partition_id = identity.partition_id,
+ path = %index_path,
+ error = %source,
+ "failed to read a sparse index entry for anchor search during recovery"
+ );
+ PartitionRecoveryError::from(IggyError::CannotReadFile)
+ })?;
let entry = IggyIndex::new(
read_u64_le(&raw, 0),
read_u64_le(&raw, 8),
@@ -2324,7 +2348,8 @@ async fn find_provable_index_anchor(
.budget
.grow_for_residue(budgeted_down_to.saturating_sub(entry.position));
budgeted_down_to = entry.position;
- if let Some(header) = header_at(identity, scanner, messages_path, entry.position)?
+ if let Some(header) =
+ header_at(identity, scanner, messages_path, entry.position).await?
&& header.partition_id == identity.partition_id as u64
&& header.base_offset == entry.offset
&& header.message_count > 0
@@ -2345,7 +2370,9 @@ async fn find_provable_index_anchor(
messages_path,
entry.position,
header.total_size(),
- )? {
+ )
+ .await?
+ {
return Ok(IndexAnchorSearch {
provable_entries: entry_index + 1,
provable_position: entry.position,
@@ -2410,6 +2437,7 @@ async fn index_is_consistent(
scratch: &mut ScanScratch,
) -> Result<IndexValidation, PartitionRecoveryError> {
let index_file = fs::File::open(index_path)
+ .await
.note_descriptor_exhaustion(|| format!("opening {index_path}"))
.map_err(|source| {
error!(
@@ -2423,6 +2451,7 @@ async fn index_is_consistent(
PartitionRecoveryError::from(IggyError::CannotReadFile)
})?;
let messages_file = fs::File::open(messages_path)
+ .await
.note_descriptor_exhaustion(|| format!("opening {messages_path}"))
.map_err(|source| {
error!(
@@ -2458,8 +2487,8 @@ async fn index_is_consistent(
let chunk_bytes =
usize::try_from(chunk_entries).unwrap_or(per_chunk_entries) * IGGY_INDEX_SIZE;
index_window.resize(chunk_bytes, 0);
- index_file
- .read_exact_at(&mut index_window[..], byte_position)
+ read_scan_window(&index_file, index_window, byte_position)
+ .await
.map_err(|source| {
error!(
stream_id = identity.stream_id,
@@ -2510,7 +2539,9 @@ async fn index_is_consistent(
}
if mappings_match
- && !log_scanner.entry_matches(entry_offset, entry_timestamp, entry_position)?
+ && !log_scanner
+ .entry_matches(entry_offset, entry_timestamp, entry_position)
+ .await?
{
warn!(
stream_id = identity.stream_id,
@@ -2545,11 +2576,12 @@ async fn index_is_consistent(
/// produces, so folding an open failure into "walked nothing" would discard
/// a healthy indexed segment's index -- or route an index-less one into
/// recover-as-empty, fencing the whole log out of service.
-fn open_messages_file(
+async fn open_messages_file(
identity: PartitionIdentity<'_>,
messages_path: &str,
) -> Result<fs::File, PartitionRecoveryError> {
fs::File::open(messages_path)
+ .await
.note_descriptor_exhaustion(|| format!("opening {messages_path}"))
.map_err(|source| {
error!(
@@ -2566,7 +2598,7 @@ fn open_messages_file(
/// The batch header at `position`, or `None` when the walk must stop there
/// (nothing decodes, or the header runs past the end of the walked file).
-fn header_at(
+async fn header_at(
identity: PartitionIdentity<'_>,
scanner: &mut FileScanner<'_>,
messages_path: &str,
@@ -2574,13 +2606,14 @@ fn header_at(
) -> Result<Option<BatchHeader>, PartitionRecoveryError> {
scanner
.peek_header(position)
+ .await
.map_err(|source| scan_read_failure(identity, messages_path, &source))
}
/// Whether the whole batch at `position` passes its batch checksum. `false`
/// when the claimed extent runs past the end of the walked file, which is
/// the torn-tail shape and not a read failure.
-fn batch_verifies(
+async fn batch_verifies(
identity: PartitionIdentity<'_>,
scanner: &mut FileScanner<'_>,
messages_path: &str,
@@ -2589,6 +2622,7 @@ fn batch_verifies(
) -> Result<bool, PartitionRecoveryError> {
Ok(scanner
.slice_at(position, total_size)
+ .await
.map_err(|source| scan_read_failure(identity, messages_path, &source))?
.is_some_and(|batch| decode_batch_slice(batch).is_ok()))
}
@@ -2718,10 +2752,8 @@ enum ProbeOutcome {
/// candidate one byte at a time, and per-candidate preads would turn one
/// damaged multi-GiB segment into a boot-length stall.
///
-/// Synchronous `std::fs` on purpose, like every mutation in this module: the
-/// boot path's runtime sizes its blocking pool at zero and recovery must not
-/// depend on `io_uring` opcode coverage. Only the sparse-index bound reads go
-/// through the async `IggyIndexReader`.
+/// Window refills use async reads so online partition recovery can share a
+/// shard with request handling, including when the runtime has no blocking pool.
struct FileScanner<'scan> {
file: &'scan fs::File,
file_len: u64,
@@ -2752,17 +2784,15 @@ impl<'scan> FileScanner<'scan> {
}
/// True when the scanner hit disk since the last call. The async scan
- /// loops yield to the reactor once per window of work on it: recovery
- /// runs in front of the bootstrap barrier with the blocking pool sized
- /// at zero, so an unyielding walk over a damaged multi-GiB chain would
- /// pin the shard core -- signal handling included -- until it finishes.
+ /// loops yield once per window of CPU work, including when reads complete
+ /// immediately, so recovery cannot monopolize the shard.
fn take_refilled(&mut self) -> bool {
std::mem::take(&mut self.refilled)
}
/// Bytes `[position, position + len)`, or `None` when they run past the
/// end of the file.
- fn slice_at(&mut self, position: u64, len: usize) -> io::Result<Option<&[u8]>> {
+ async fn slice_at(&mut self, position: u64, len: usize) -> io::Result<Option<&[u8]>> {
let Some(end) = position.checked_add(len as u64) else {
return Ok(None);
};
@@ -2774,7 +2804,7 @@ impl<'scan> FileScanner<'scan> {
// Callers only pass lengths from headers that already passed the
// plausibility cap, which is what bounds this resize.
self.spill.resize(len, 0);
- self.file.read_exact_at(&mut self.spill[..], position)?;
+ read_scan_window(self.file, self.spill, position).await?;
self.refilled = true;
return Ok(Some(&self.spill[..]));
}
@@ -2795,8 +2825,7 @@ impl<'scan> FileScanner<'scan> {
usize::try_from((self.file_len - window_start).min(SCAN_WINDOW_CAPACITY as u64))
.unwrap_or(SCAN_WINDOW_CAPACITY);
self.window.resize(fill, 0);
- self.file
- .read_exact_at(&mut self.window[..], window_start)?;
+ read_scan_window(self.file, self.window, window_start).await?;
self.window_start = window_start;
self.refilled = true;
}
@@ -2813,8 +2842,8 @@ impl<'scan> FileScanner<'scan> {
/// so treating the header as undecodable is verdict-identical to reading
/// the claimed bytes and failing the verify, and it keeps one bit-flipped
/// length field from driving a claimed-size allocation and read.
- fn peek_header(&mut self, position: u64) -> io::Result<Option<BatchHeader>> {
- let Some(bytes) = self.slice_at(position, COMMAND_HEADER_SIZE)? else {
+ async fn peek_header(&mut self, position: u64) -> io::Result<Option<BatchHeader>> {
+ let Some(bytes) = self.slice_at(position, COMMAND_HEADER_SIZE).await? else {
return Ok(None);
};
Ok(BatchHeader::decode(bytes)
@@ -2860,7 +2889,7 @@ impl<'scan> FileScanner<'scan> {
// first candidate starts one past them.
let mut candidate = damage_position.saturating_add(1);
while candidate.saturating_add(header_len) <= self.file_len {
- self.fill_window_at(candidate)?;
+ self.fill_window_at(candidate).await?;
let window_end = self.window_start + self.window.len() as u64;
while candidate.saturating_add(header_len) <= window_end {
if !self.budget.charge_candidate() {
@@ -2886,7 +2915,7 @@ impl<'scan> FileScanner<'scan> {
if !self.budget.charge_verify(total_size as u64) {
return Ok(ProbeOutcome::BudgetExhausted);
}
- let batch = self.verify_slice(candidate, total_size)?;
+ let batch = self.verify_slice(candidate, total_size).await?;
if decode_batch_slice(batch).is_ok() {
return Ok(ProbeOutcome::Survivor {
position: candidate,
@@ -2914,7 +2943,7 @@ impl<'scan> FileScanner<'scan> {
/// Anchors the window at `position` unless the header there already sits
/// inside it. The probe's outer loop refills through this, so its
/// windows advance strictly forward.
- fn fill_window_at(&mut self, position: u64) -> io::Result<()> {
+ async fn fill_window_at(&mut self, position: u64) -> io::Result<()> {
let window_end = self.window_start + self.window.len() as u64;
if position >= self.window_start
&& position.saturating_add(COMMAND_HEADER_SIZE as u64) <= window_end
@@ -2924,7 +2953,7 @@ impl<'scan> FileScanner<'scan> {
let fill = usize::try_from((self.file_len - position).min(SCAN_WINDOW_CAPACITY as u64))
.unwrap_or(SCAN_WINDOW_CAPACITY);
self.window.resize(fill, 0);
- self.file.read_exact_at(&mut self.window[..], position)?;
+ read_scan_window(self.file, self.window, position).await?;
self.window_start = position;
self.refilled = true;
Ok(())
@@ -2935,7 +2964,7 @@ impl<'scan> FileScanner<'scan> {
/// else is one direct read into the spill buffer. The caller bounds
/// `len` against the file and the plausibility cap before calling, which
/// is what bounds the spill's growth.
- fn verify_slice(&mut self, position: u64, len: usize) -> io::Result<&[u8]> {
+ async fn verify_slice(&mut self, position: u64, len: usize) -> io::Result<&[u8]> {
let window_end = self.window_start + self.window.len() as u64;
let end = position.saturating_add(len as u64);
if position >= self.window_start && end <= window_end {
@@ -2945,7 +2974,7 @@ impl<'scan> FileScanner<'scan> {
return Ok(&self.window[at..at + len]);
}
self.spill.resize(len, 0);
- self.file.read_exact_at(&mut self.spill[..], position)?;
+ read_scan_window(self.file, self.spill, position).await?;
self.refilled = true;
Ok(&self.spill[..])
}
@@ -2957,6 +2986,16 @@ fn push_index_entry(rebuilt_index: &mut Vec<u8>, offset: u64, timestamp: u64, po
rebuilt_index.extend_from_slice(&position.to_le_bytes());
}
+async fn read_scan_window(file: &fs::File, window: &mut Vec<u8>, position: u64) -> io::Result<()> {
+ let length = window.len();
+ let (result, bytes) = file
+ .read_exact_at(std::mem::take(window).slice(..length), position)
+ .await
+ .into();
+ *window = bytes.into_inner();
+ result
+}
+
fn read_u64_le(bytes: &[u8], at: usize) -> u64 {
let mut raw = [0u8; 8];
raw.copy_from_slice(&bytes[at..at + 8]);
@@ -2973,6 +3012,7 @@ mod tests {
IggyMessage, IggyMessageHeader, IggyMessages, SendMessagesOwned, calculate_batch_checksum,
};
use server_common::sharding::IggyNamespace;
+ use std::fs;
use std::os::unix::fs::symlink;
use tempfile::{TempDir, tempdir};
@@ -3205,29 +3245,32 @@ mod tests {
.await
}
- #[compio::test]
- async fn given_torn_log_tail_when_recovering_should_truncate_files_to_walked_bounds() {
- let tmp = tempdir().expect("tempdir");
- let config = test_config(&tmp);
- prepare_partition_dir(&config);
- let mut log = encoded_batch(0, 3);
- let valid_len = log.len() as u64;
- log.extend_from_slice(&GARBAGE);
- let (messages_path, index_path) = write_segment(&config, 0, &log, &index_entry(0, 0));
-
- let recovered = recover(&config).await.expect("recover torn-tail segment");
-
- assert_eq!(recovered.len(), 1);
- let segment = &recovered[0].segment;
- assert_eq!(segment.end_offset, 2);
- assert_eq!(segment.size, IggyByteSize::from(valid_len));
- assert_eq!(segment.current_position, valid_len);
- assert_eq!(
- len_of(&messages_path),
- valid_len,
- "torn tail bytes must be gone from disk"
- );
- assert_eq!(len_of(&index_path), IGGY_INDEX_SIZE as u64);
+ #[test]
+ fn given_torn_log_tail_when_recovering_should_truncate_files_to_walked_bounds() {
+ let runtime = server_common::executor::create_shard_executor().unwrap();
+ runtime.block_on(async {
+ let tmp = tempdir().expect("tempdir");
+ let config = test_config(&tmp);
+ prepare_partition_dir(&config);
+ let mut log = encoded_batch(0, 3);
+ let valid_len = log.len() as u64;
+ log.extend_from_slice(&GARBAGE);
+ let (messages_path, index_path) = write_segment(&config, 0, &log, &index_entry(0, 0));
+
+ let recovered = recover(&config).await.expect("recover torn-tail segment");
+
+ assert_eq!(recovered.len(), 1);
+ let segment = &recovered[0].segment;
+ assert_eq!(segment.end_offset, 2);
+ assert_eq!(segment.size, IggyByteSize::from(valid_len));
+ assert_eq!(segment.current_position, valid_len);
+ assert_eq!(
+ len_of(&messages_path),
+ valid_len,
+ "torn tail bytes must be gone from disk"
+ );
+ assert_eq!(len_of(&index_path), IGGY_INDEX_SIZE as u64);
+ });
}
#[compio::test]
@@ -4419,7 +4462,9 @@ mod tests {
log.extend_from_slice(&GARBAGE);
let messages_path = tmp.path().join("00000000000000000000.log");
fs::write(&messages_path, &log).expect("write log fixture");
- let file = fs::File::open(&messages_path).expect("open log fixture");
+ let file = compio::fs::File::open(&messages_path)
+ .await
+ .expect("open log fixture");
let partition_path = tmp.path().to_string_lossy().into_owned();
let identity = PartitionIdentity {
partition_path: &partition_path,
@@ -4478,7 +4523,9 @@ mod tests {
let log = GARBAGE.to_vec();
let messages_path = tmp.path().join("00000000000000000000.log");
fs::write(&messages_path, &log).expect("write log fixture");
- let file = fs::File::open(&messages_path).expect("open log fixture");
+ let file = compio::fs::File::open(&messages_path)
+ .await
+ .expect("open log fixture");
let partition_path = tmp.path().to_string_lossy().into_owned();
let identity = PartitionIdentity {
partition_path: &partition_path,
diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs
index c03bbdb96..df2c29cbd 100644
--- a/core/partitions/src/state_transfer.rs
+++ b/core/partitions/src/state_transfer.rs
@@ -1864,19 +1864,18 @@ fn staging_paths(partition_dir: &str, start_offset: u64) -> (PathBuf, PathBuf) {
/// Every entry of one partition directory, as paths.
///
-/// BLOCKING `read_dir` on the pump: compio-fs 0.12 exposes no async directory
-/// walk, and `spawn_blocking` is not an escape either -- the shard executors run
-/// `thread_pool_limit(0)`. Bounded by the entry count of ONE partition directory,
-/// but it is a real stall (and under the write lock at the converge site), so it
-/// stays recorded rather than hidden.
-///
/// Enumeration only: callers keep their own predicates and their own
/// error policies (propagate / silent skip / log-and-fail), which is what
/// `sweep_staging_except`'s do-not-widen warning depends on.
-fn segment_dir_entries(partition_dir: &str) -> std::io::Result<impl Iterator<Item = PathBuf>> {
- Ok(std::fs::read_dir(partition_dir)?
- .flatten()
- .map(|entry| entry.path()))
+pub(crate) async fn segment_dir_entries(partition_dir: &str) -> std::io::Result<Vec<PathBuf>> {
+ let partition_dir = partition_dir.to_owned();
+ server_common::fs_utils::run_blocking("iggy-segment-scan", move || {
+ Ok(std::fs::read_dir(partition_dir)?
+ .flatten()
+ .map(|entry| entry.path())
+ .collect())
+ })
+ .await
}
/// Remove physical tails outside the logical segment list after draining the WAL.
@@ -1995,7 +1994,7 @@ pub async fn quarantine_partition_files(
"a thousand fenced copies of this partition already exist",
));
};
- for path in segment_dir_entries(partition_dir)? {
+ for path in segment_dir_entries(partition_dir).await? {
let quarantined = path.to_str().is_some_and(|path| {
[".log", ".index", STAGING_SUFFIX, ANCHOR_SUFFIX]
.iter()
@@ -2048,7 +2047,7 @@ pub async fn quarantine_partition_files(
/// on the partition -- worst at the reuse scan, which runs at descriptor-accept
/// on a serving partition.
pub(crate) async fn sweep_staging_except(partition_dir: &str, keep: &HashSet<&Path>) {
- let Ok(entries) = segment_dir_entries(partition_dir) else {
+ let Ok(entries) = segment_dir_entries(partition_dir).await else {
return;
};
for path in entries {
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index 75382368b..4ec143ac3 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -89,6 +89,7 @@ systemd = ["dep:sd-notify"]
[dependencies]
ahash = { workspace = true }
+arc-swap = { workspace = true }
argon2 = { workspace = true }
async-channel = { workspace = true }
async_zip = { workspace = true }
diff --git a/core/server/src/boot/mod.rs b/core/server/src/boot/mod.rs
index f3d81704e..cf7d7f1ea 100644
--- a/core/server/src/boot/mod.rs
+++ b/core/server/src/boot/mod.rs
@@ -303,8 +303,8 @@ pub fn bootstrap(
crate::sysinfo_probe::init_stats_data_path(config.get_system_path().into());
// Before the shards, so the first count is there when a client connects.
let sysinfo_print_interval = config.logging.sysinfo_print_interval.get_duration();
- if let Err(error) = crate::sysinfo_probe::start_open_files_scan(sysinfo_print_interval) {
- warn!(error = %error, "cannot start the open-files scan thread, so open_files_count stays 0");
+ if let Err(error) = crate::sysinfo_probe::start_system_stats_sampler(sysinfo_print_interval) {
+ warn!(error = %error, "cannot start the system stats sampler; keeping the initial sample");
}
let (assignments, total_shards) = resolve_shard_assignments(&config.sharding)?;
let shards_count = assignments.len();
diff --git a/core/server/src/partition_helpers.rs b/core/server/src/partition_helpers.rs
index 8f28e6e75..08795fe66 100644
--- a/core/server/src/partition_helpers.rs
+++ b/core/server/src/partition_helpers.rs
@@ -53,6 +53,7 @@ use partitions::{
load_persisted_segments, load_persisted_segments_with_checkpoint,
};
use server_common::fatal::is_descriptor_exhaustion;
+use server_common::fs_utils::run_blocking;
use server_common::sharding::IggyNamespace;
use std::path::{Path, PathBuf};
use std::rc::Rc;
@@ -278,8 +279,15 @@ pub async fn load_partition_or_fence(
}
Ok(Some(_)) => false,
Ok(None) => {
- match std::fs::read_dir(&directory).and_then(|mut entries| entries.next().transpose()) {
- Ok(Some(_)) => warn!(
+ let path = directory.clone();
+ let nonempty = run_blocking("iggy-partition-scan", move || {
+ std::fs::read_dir(path)
+ .and_then(|mut entries| entries.next().transpose())
+ .map(|entry| entry.is_some())
+ })
+ .await;
+ match nonempty {
+ Ok(true) => warn!(
stream_id,
topic_id,
partition_id = partition_metadata.id,
@@ -289,7 +297,7 @@ pub async fn load_partition_or_fence(
if this namespace was deleted and recreated, delete and recreate it again \
only if its messages can be discarded; see core/server/README.md"
),
- Ok(None) => {}
+ Ok(false) => {}
Err(source) if source.kind() == std::io::ErrorKind::NotFound => {}
Err(source) => return Err(revision_io(source)),
}
@@ -2655,11 +2663,6 @@ mod tests {
std::fs::File::create(partitions_config.get_messages_path(1, 1, 0, start_offset))
.expect("empty segment log");
}
- // The rebuild's claim is this group's first superblock write, so it
- // targets slot A. A directory where its temp file goes fails the atomic
- // replace and nothing else: the slot reads still find the store empty,
- // and the quarantine moves segment files only.
- std::fs::create_dir(Path::new(&dir).join("superblock.a.tmp")).expect("block slot A");
drop(
journal::PartitionPrepareJournal::open(
&Path::new(&dir).join("prepares-0"),
@@ -2672,24 +2675,40 @@ mod tests {
let stats = Arc::new(PartitionStats::default());
let partitions = solo_partitions(&config);
- let loaded = load_partition_or_fence(
+ let partition_metadata = Partition::new(0, namespace.inner(), IggyTimestamp::now(), 0, 0);
+ let mut loaded = Box::pin(load_partition_or_fence(
&config,
namespace,
Arc::clone(&stats),
- &Partition::new(0, namespace.inner(), IggyTimestamp::now(), 0, 0),
+ &partition_metadata,
TopicRuntimeOptions::default(),
CLUSTER,
0,
1,
Rc::new(IggyMessageBus::new(0)),
&partitions,
- )
- .await;
+ ));
+ // Segment provisioning finishes before the claim's detached file-open
+ // task can run. Removing its directory here fails only that claim.
+ loop {
+ assert!(
+ futures::poll!(&mut loaded).is_pending(),
+ "the loader must reach the claim before completing"
+ );
+ if stats.segments_count_inconsistent() == 1 {
+ break;
+ }
+ server_common::yield_to_reactor().await;
+ }
+ let unavailable = root.path().join("unavailable-partition");
+ std::fs::rename(&dir, &unavailable).expect("make the claim's directory unavailable");
+ let loaded = loaded.await;
+ std::fs::rename(&unavailable, &dir).expect("restore the partition directory");
match loaded {
Err(ServerError::PartitionOffsetReservationClaim { .. }) => {}
Err(other) => panic!("expected the claim's own refusal, got {other}"),
- Ok(Some(_)) => panic!("the planted directory must fail the rebuild's claim"),
+ Ok(Some(_)) => panic!("the missing directory must fail the rebuild's claim"),
Ok(None) => panic!("a refused claim must reach the caller, not be absorbed here"),
}
assert!(
diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs
index 57ee07b72..5687e24f1 100644
--- a/core/server/src/partition_reconciler.rs
+++ b/core/server/src/partition_reconciler.rs
@@ -738,7 +738,7 @@ async fn reconcile_additions(
let partition_dir =
ctx.config
.get_partition_path(ns.stream_id(), ns.topic_id(), ns.partition_id());
- let prior_life_on_disk = std::fs::metadata(&partition_dir).is_ok();
+ let prior_life_on_disk = compio::fs::metadata(&partition_dir).await.is_ok();
// The target was snapshotted before this read, so a delete plus a
// recreate of the same slab keys can commit in between. Everything
diff --git a/core/server/src/responses.rs b/core/server/src/responses.rs
index ac95c4b4c..d3fdf91f6 100644
--- a/core/server/src/responses.rs
+++ b/core/server/src/responses.rs
@@ -27,7 +27,7 @@ use crate::namespace::ensure_topic_exists;
use crate::reply_frame::{build_empty_reply, build_reply_from_bytes};
use crate::session_manager::SessionManager;
use crate::shell::{ShellBus, ShellShard};
-use crate::sysinfo_probe::{probe_system_stats, stats_disk_space};
+use crate::sysinfo_probe::probe_system_stats;
use crate::wire::{transport_kind_to_wire, usize_to_u32};
use bytes::Bytes;
use consensus::MetadataHandle;
@@ -515,7 +515,6 @@ where
)?;
let system = probe_system_stats();
- let (free_disk_space, total_disk_space) = stats_disk_space();
Ok(StatsResponse {
process_id: system.process_id,
cpu_usage: system.cpu_usage,
@@ -543,8 +542,8 @@ where
iggy_server_semver: crate::SEMANTIC_VERSION.get_numeric_version().ok(),
cache_metrics: Vec::new(),
threads_count: system.threads_count,
- free_disk_space,
- total_disk_space,
+ free_disk_space: system.free_disk_space,
+ total_disk_space: system.total_disk_space,
open_files_count: system.open_files_count,
open_files_limit: system.open_files_limit,
})
diff --git a/core/server/src/sysinfo_printer.rs b/core/server/src/sysinfo_printer.rs
index 0b53cdac1..f6fc04da0 100644
--- a/core/server/src/sysinfo_printer.rs
+++ b/core/server/src/sysinfo_printer.rs
@@ -22,7 +22,7 @@
//! gather.
use crate::shell::ServerShard;
-use crate::sysinfo_probe::{SystemStats, stats_disk_space};
+use crate::sysinfo_probe::{SystemStats, probe_system_stats};
use consensus::MetadataHandle;
use iggy_common::IggyByteSize;
use metadata::impls::metadata::StreamsFrontend;
@@ -30,29 +30,23 @@ use shard::Receiver;
use std::fmt;
use std::rc::Rc;
use std::time::Duration;
-use sysinfo::System as SysinfoSystem;
use tracing::level_filters::LevelFilter;
use tracing::{info, trace};
/// Run the printer until `stop` fires, logging one line every `interval`.
pub async fn run_sysinfo_printer(shard: Rc<ServerShard>, stop: Receiver<()>, interval: Duration) {
info!("System info logger is enabled, OS info will be printed every: {interval:?}");
- // Not the shared `GetStats` sampler: a tick would reset its CPU window, and
- // a `GetStats` on shard 0 right after a tick would show CPU near zero.
- // Sampled once now, so the first line covers a full interval.
- let mut system = SysinfoSystem::new();
- SystemStats::capture(&mut system);
loop {
// `Ok(_)`: stop signalled -> exit. `Err(_)`: interval elapsed -> print.
match compio::time::timeout(interval, stop.recv()).await {
Ok(_) => break,
- Err(_) => print_sysinfo(&shard, &mut system).await,
+ Err(_) => print_sysinfo(&shard).await,
}
}
trace!(shard = shard.id, "sysinfo printer exited");
}
-async fn print_sysinfo(shard: &Rc<ServerShard>, system: &mut SysinfoSystem) {
+async fn print_sysinfo(shard: &Rc<ServerShard>) {
// The global max level, not `tracing::enabled!`: the logger's idle
// OpenTelemetry layers veto every `enabled` query, so that is always false.
if LevelFilter::current() < LevelFilter::INFO {
@@ -60,9 +54,11 @@ async fn print_sysinfo(shard: &Rc<ServerShard>, system: &mut SysinfoSystem) {
}
let clients_count = shard.count_all_clients().await;
let (messages_size_bytes, messages_count) = messages_totals(shard);
- let (free_disk_space, total_disk_space) = stats_disk_space();
+ let system = probe_system_stats();
+ let free_disk_space = system.free_disk_space;
+ let total_disk_space = system.total_disk_space;
let line = SysinfoLine {
- system: SystemStats::capture(system),
+ system,
messages_size_bytes,
messages_count,
clients_count,
@@ -165,6 +161,8 @@ mod tests {
os_name: String::new(),
os_version: String::new(),
kernel_version: String::new(),
+ free_disk_space: 0,
+ total_disk_space: 0,
},
messages_size_bytes: 0,
messages_count: 42,
diff --git a/core/server/src/sysinfo_probe.rs b/core/server/src/sysinfo_probe.rs
index 8f8576f8c..d12c76332 100644
--- a/core/server/src/sysinfo_probe.rs
+++ b/core/server/src/sysinfo_probe.rs
@@ -19,22 +19,20 @@
//! sampling of this process, the cached host identity, and disk usage of the
//! volume holding the data directory.
+use arc_swap::ArcSwap;
use nix::sys::resource::{Resource, getrlimit};
-use std::cell::RefCell;
use std::io;
use std::path::PathBuf;
-use std::sync::OnceLock;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, LazyLock, OnceLock};
use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
use sysinfo::System as SysinfoSystem;
use system_stats::{SystemProbe, count_open_files, count_open_files_without_scan};
-/// Process- and host-level portion of the stats reply, probed via `sysinfo`.
-/// These describe the whole process, not shard or metadata state, so any one
-/// shard can serve them without aggregation. The CPU fields are deltas since
-/// the previous refresh of the probed `System`, so they vary by serving shard
-/// (a shard's first probe reports zero CPU).
+/// The last completed process-wide sample. CPU deltas cover the sampler interval.
+/// Slow probes leave the previous sample available without delaying shard readers.
+#[derive(Clone, Default)]
pub struct SystemStats {
pub process_id: u32,
pub cpu_usage: f32,
@@ -47,8 +45,7 @@ pub struct SystemStats {
pub read_bytes: u64,
pub written_bytes: u64,
pub threads_count: u32,
- /// 0 when unknown. See [`start_open_files_scan`] for where the count
- /// comes from without a kernel count.
+ /// 0 when unknown; directory-scan fallbacks refresh at most every ten seconds.
pub open_files_count: u64,
/// Soft `RLIMIT_NOFILE`, the ceiling `open()` fails at. 0 when unknown.
pub open_files_limit: u64,
@@ -56,19 +53,11 @@ pub struct SystemStats {
pub os_name: String,
pub os_version: String,
pub kernel_version: String,
+ pub free_disk_space: u64,
+ pub total_disk_space: u64,
}
-thread_local! {
- // `cpu_usage` is a delta since the previous refresh, so the sampled
- // `System` is kept alive across `GetStats` calls (a freshly created one
- // reports zero CPU). Mirrors the legacy shard-0 stats path.
- static SYSINFO: RefCell<Option<SysinfoSystem>> = const { RefCell::new(None) };
-}
-
-/// Host / OS identity is process-static (unlike the per-call CPU and memory
-/// samples), so probe it once and clone from the cache on each `GetStats`
-/// rather than re-querying sysinfo every call. Process-global, so a `OnceLock`
-/// fits better than the per-thread [`SYSINFO`] cell.
+/// Host identity is sampled only once, before the shard threads start.
struct HostIdentity {
hostname: String,
os_name: String,
@@ -98,84 +87,73 @@ static HOST_IDENTITY: OnceLock<HostIdentity> = OnceLock::new();
/// bootstrap.
static STATS_DATA_PATH: OnceLock<PathBuf> = OnceLock::new();
-/// Last open-descriptor count from [`start_open_files_scan`], 0 before the
-/// first. Process-global because the scan has a thread of its own and any
-/// shard can serve `GetStats`.
-static PUBLISHED_OPEN_FILES_COUNT: AtomicU64 = AtomicU64::new(0);
+static PUBLISHED_STATS: LazyLock<ArcSwap<SystemStats>> =
+ LazyLock::new(|| ArcSwap::from_pointee(SystemStats::default()));
+static SAMPLER_STARTED: AtomicBool = AtomicBool::new(false);
-/// The longest period of the scan thread, and so the oldest the published
-/// count gets.
+const SYSTEM_STATS_INTERVAL: Duration = Duration::from_secs(1);
const OPEN_FILES_SCAN_INTERVAL: Duration = Duration::from_secs(10);
-/// Capture the configured data directory for `GetStats` disk reporting.
-/// Idempotent: only the first call (process bootstrap) takes effect.
+/// Capture the configured data directory before starting the sampler.
pub fn init_stats_data_path(path: PathBuf) {
let _ = STATS_DATA_PATH.set(path);
}
-/// Free and total bytes of the volume holding the configured data directory,
-/// `(0, 0)` before bootstrap or on a probe error.
-pub fn stats_disk_space() -> (u64, u64) {
- STATS_DATA_PATH.get().map_or((0, 0), |path| {
- (
- fs2::available_space(path).unwrap_or(0),
- fs2::total_space(path).unwrap_or(0),
- )
- })
-}
-
-/// Start the thread that counts open descriptors for `GetStats` and the
-/// sysinfo line where the kernel keeps no count. Where the kernel keeps one,
-/// start nothing.
-///
-/// The thread scans every [`OPEN_FILES_SCAN_INTERVAL`], or every
-/// `sysinfo_print_interval` when that is shorter and not zero, so each line
-/// shows a count from its own interval. The scan blocks for a time that grows
-/// with the count, and a shard must not block. `spawn_blocking` is not
-/// available, because shards run without the blocking pool of the runtime.
-/// The thread runs until the process exits, also while the printer is
-/// disabled.
-pub fn start_open_files_scan(sysinfo_print_interval: Duration) -> io::Result<()> {
- if count_open_files_without_scan().is_some() {
+/// Start one process-lifetime sampler before any shard threads exist.
+/// Samples refresh every second, or the printer interval when shorter. The
+/// descriptor-scan fallback refreshes every ten seconds or printer interval.
+/// Sampling time extends these intervals; readers always get the last completed sample.
+pub fn start_system_stats_sampler(sysinfo_print_interval: Duration) -> io::Result<()> {
+ if SAMPLER_STARTED.swap(true, Ordering::AcqRel) {
return Ok(());
}
let interval = if sysinfo_print_interval.is_zero() {
+ SYSTEM_STATS_INTERVAL
+ } else {
+ sysinfo_print_interval.min(SYSTEM_STATS_INTERVAL)
+ };
+ let scan_interval = if sysinfo_print_interval.is_zero() {
OPEN_FILES_SCAN_INTERVAL
} else {
sysinfo_print_interval.min(OPEN_FILES_SCAN_INTERVAL)
};
- thread::Builder::new()
- .name("iggy-open-files".to_owned())
+ let mut system = SysinfoSystem::new();
+ let initial = SystemStats::capture(&mut system, 0, true);
+ let mut open_files_count = initial.open_files_count;
+ PUBLISHED_STATS.store(Arc::new(initial));
+ let spawned = thread::Builder::new()
+ .name("iggy-system-stats".to_owned())
.spawn(move || {
+ let mut last_scan = Instant::now();
loop {
- publish_open_files_count(count_open_files());
thread::sleep(interval);
+ let scan = last_scan.elapsed() >= scan_interval;
+ let sample = SystemStats::capture(&mut system, open_files_count, scan);
+ open_files_count = sample.open_files_count;
+ if scan {
+ last_scan = Instant::now();
+ }
+ PUBLISHED_STATS.store(Arc::new(sample));
}
- })
- .map(drop)
-}
-
-/// A failed scan keeps the previous count. Before Linux 6.2, the scan needs a
-/// free descriptor, so it fails when the table is full, and 0 would then read
-/// as unknown.
-fn publish_open_files_count(count: Option<u64>) {
- if let Some(count) = count {
- PUBLISHED_OPEN_FILES_COUNT.store(count, Ordering::Relaxed);
+ });
+ if spawned.is_err() {
+ SAMPLER_STARTED.store(false, Ordering::Release);
}
+ spawned.map(drop)
}
-/// Probe through this thread's [`SYSINFO`], the `GetStats` path.
+/// Read the last completed sample without filesystem access or a blocking lock.
pub fn probe_system_stats() -> SystemStats {
- SYSINFO
- .with_borrow_mut(|slot| SystemStats::capture(slot.get_or_insert_with(SysinfoSystem::new)))
+ PUBLISHED_STATS.load().as_ref().clone()
}
impl SystemStats {
- /// Probe through `sys`. A caller that keeps its own `sys` gets CPU deltas
- /// over its own interval, and does not reset the window of `GetStats`.
- pub fn capture(sys: &mut SysinfoSystem) -> Self {
+ fn capture(sys: &mut SysinfoSystem, previous_open_files: u64, scan_open_files: bool) -> Self {
let host = HOST_IDENTITY.get_or_init(HostIdentity::probe);
let probe = SystemProbe::capture(sys);
+ let disk = STATS_DATA_PATH
+ .get()
+ .and_then(|path| fs2::statvfs(path).ok());
Self {
process_id: probe.process_id,
cpu_usage: probe.cpu_usage,
@@ -192,12 +170,15 @@ impl SystemStats {
written_bytes: probe.written_bytes,
threads_count: probe.threads_count,
open_files_count: count_open_files_without_scan()
- .unwrap_or_else(|| PUBLISHED_OPEN_FILES_COUNT.load(Ordering::Relaxed)),
+ .or_else(|| scan_open_files.then(count_open_files).flatten())
+ .unwrap_or(previous_open_files),
open_files_limit: getrlimit(Resource::RLIMIT_NOFILE).map_or(0, |(soft, _)| soft),
hostname: host.hostname.clone(),
os_name: host.os_name.clone(),
os_version: host.os_version.clone(),
kernel_version: host.kernel_version.clone(),
+ free_disk_space: disk.as_ref().map_or(0, fs2::FsStats::available_space),
+ total_disk_space: disk.as_ref().map_or(0, fs2::FsStats::total_space),
}
}
}
@@ -207,21 +188,19 @@ mod tests {
use super::*;
#[test]
- fn probe_system_stats_reports_this_process_and_host_memory() {
- let stats = probe_system_stats();
- // Straight from `sysinfo`, independent of shard state: the pid is our
- // own and any host the test runs on has nonzero total memory. A zero
- // here means the probe wired nothing (the pre-fix stubbed literal).
- assert_eq!(stats.process_id, std::process::id());
- assert!(stats.total_memory > 0);
- assert!(!stats.hostname.is_empty());
- }
-
- #[test]
- fn given_failed_scan_when_publishing_should_keep_previous_count() {
- publish_open_files_count(Some(42));
- publish_open_files_count(None);
-
- assert_eq!(PUBLISHED_OPEN_FILES_COUNT.load(Ordering::Relaxed), 42);
+ fn given_completed_sample_when_reading_stats_should_reuse_it() {
+ let mut system = SysinfoSystem::new();
+ let sample = SystemStats::capture(&mut system, 0, true);
+ assert_eq!(sample.process_id, std::process::id());
+ assert!(sample.total_memory > 0);
+ assert!(!sample.hostname.is_empty());
+ PUBLISHED_STATS.store(Arc::new(sample));
+
+ let published = PUBLISHED_STATS.load_full();
+ let first = probe_system_stats();
+ let second = probe_system_stats();
+ assert_eq!(first.read_bytes, second.read_bytes);
+ assert_eq!(first.memory_usage, published.memory_usage);
+ assert!(Arc::ptr_eq(&published, &PUBLISHED_STATS.load_full()));
}
}
diff --git a/core/server_common/src/fs_utils.rs b/core/server_common/src/fs_utils.rs
index c64090c56..db75b15d1 100644
--- a/core/server_common/src/fs_utils.rs
+++ b/core/server_common/src/fs_utils.rs
@@ -16,10 +16,14 @@
// under the License.
use compio::fs;
+use futures::channel::oneshot;
+use futures::lock::Mutex;
use std::io;
use std::path::{Path, PathBuf};
use tracing::warn;
+use crate::fatal::NoteDescriptorExhaustion;
+
#[cfg(target_os = "linux")]
use nix::fcntl::{FallocateFlags, fallocate};
@@ -54,7 +58,7 @@ impl DirEntry {
/// Reserve segment space without changing its contents or logical length.
/// Unsupported or failed reservations fall back to buffered allocation.
#[cfg(target_os = "linux")]
-pub fn preallocate_file(file: &fs::File, file_path: &Path, len: u64) {
+pub async fn preallocate_file(file: &fs::File, file_path: &Path, len: u64) {
let Ok(len) = i64::try_from(len) else {
warn!(
target: "iggy.partitions.storage",
@@ -65,9 +69,19 @@ pub fn preallocate_file(file: &fs::File, file_path: &Path, len: u64) {
return;
};
- // Shard runtimes disable the worker pool, so this opt-in reservation runs
- // inline. Slow filesystem allocation stalls the shard until it returns.
- if let Err(error) = fallocate(file, FallocateFlags::FALLOC_FL_KEEP_SIZE, 0, len) {
+ let reservation = async {
+ let descriptor = std::os::fd::AsFd::as_fd(file)
+ .try_clone_to_owned()
+ .note_descriptor_exhaustion(|| {
+ "duplicating a file descriptor to preallocate".to_owned()
+ })?;
+ run_blocking("iggy-file-preallocate", move || {
+ fallocate(descriptor, FallocateFlags::FALLOC_FL_KEEP_SIZE, 0, len)
+ .map_err(io::Error::from)
+ })
+ .await
+ };
+ if let Err(error) = reservation.await {
warn!(
target: "iggy.partitions.storage",
file = %file_path.display(),
@@ -80,7 +94,7 @@ pub fn preallocate_file(file: &fs::File, file_path: &Path, len: u64) {
/// Reserve segment space when supported, without changing its logical length.
#[cfg(not(target_os = "linux"))]
-pub fn preallocate_file(_file: &fs::File, file_path: &Path, _len: u64) {
+pub async fn preallocate_file(_file: &fs::File, file_path: &Path, _len: u64) {
PREALLOCATION_UNAVAILABLE.call_once(|| {
warn!(
target: "iggy.partitions.storage",
@@ -95,9 +109,12 @@ pub fn preallocate_file(_file: &fs::File, file_path: &Path, _len: u64) {
/// safe deletion (contents before containers).
/// Symlinks are treated as files and not followed.
pub async fn walk_dir(root: impl AsRef<Path>) -> io::Result<Vec<DirEntry>> {
- let root = root.as_ref();
+ let root = root.as_ref().to_path_buf();
+ run_blocking("iggy-directory-walk", move || walk_dir_blocking(&root)).await
+}
- let metadata = fs::metadata(root).await?;
+fn walk_dir_blocking(root: &Path) -> io::Result<Vec<DirEntry>> {
+ let metadata = std::fs::metadata(root)?;
if !metadata.is_dir() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
@@ -118,7 +135,7 @@ pub async fn walk_dir(root: impl AsRef<Path>) -> io::Result<Vec<DirEntry>> {
for entry in std::fs::read_dir(¤t_dir)? {
let entry = entry?;
let entry_path = entry.path();
- let metadata = fs::symlink_metadata(&entry_path).await?;
+ let metadata = entry.file_type()?;
if metadata.is_dir() {
stack.push(entry_path);
@@ -148,3 +165,91 @@ pub async fn remove_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
}
Ok(())
}
+
+/// Truncate the open inode without requiring IORING_OP_FTRUNCATE support.
+///
+/// # Errors
+/// Returns descriptor duplication or filesystem errors.
+pub async fn truncate_file(file: &fs::File, length: u64) -> io::Result<()> {
+ let descriptor = std::os::fd::AsFd::as_fd(file)
+ .try_clone_to_owned()
+ .note_descriptor_exhaustion(|| "duplicating a file descriptor to truncate".to_owned())?;
+ run_blocking("iggy-file-truncate", move || {
+ std::fs::File::from(descriptor).set_len(length)
+ })
+ .await
+}
+
+/// Run filesystem work unsupported by io_uring on a process-wide bounded worker.
+/// The operation owns its state and continues to completion if the caller is dropped.
+///
+/// # Errors
+/// Returns thread creation, operation, or interrupted-worker errors.
+pub async fn run_blocking<T: Send + 'static>(
+ name: &'static str,
+ operation: impl FnOnce() -> io::Result<T> + Send + 'static,
+) -> io::Result<T> {
+ // Cancellation must not admit another operation while this one owns filesystem state.
+ static WORKER: Mutex<()> = Mutex::new(());
+ let permit = WORKER.lock().await;
+ let (sender, receiver) = oneshot::channel();
+ std::thread::Builder::new()
+ .name(name.to_owned())
+ .spawn(move || {
+ let _permit = permit;
+ let _ = sender.send(operation());
+ })?;
+ receiver
+ .await
+ .map_err(|_| io::Error::other(format!("{name} stopped")))?
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::executor::create_shard_executor;
+
+ #[test]
+ fn given_nested_directory_with_symlink_when_removing_should_preserve_external_target() {
+ let directory = tempfile::tempdir().unwrap();
+ let root = directory.path().join("partition");
+ let child = root.join("nested");
+ let target = directory.path().join("external");
+ std::fs::create_dir_all(&child).unwrap();
+ std::fs::write(child.join("segment.log"), b"messages").unwrap();
+ std::fs::write(&target, b"keep").unwrap();
+ std::os::unix::fs::symlink(&target, root.join("link")).unwrap();
+ let runtime = create_shard_executor().unwrap();
+ runtime.block_on(async {
+ let entries = walk_dir(&root).await.unwrap();
+ assert_eq!(entries.last().unwrap().path, root);
+ remove_dir_all(&root).await.unwrap();
+ });
+ assert!(!root.exists());
+ assert_eq!(std::fs::read(&target).unwrap(), b"keep");
+ }
+
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn given_segment_contents_when_preallocating_without_pool_should_preserve_bytes_and_length() {
+ const RESERVATION_BYTES: u64 = 1024 * 1024;
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("segment.log");
+ std::fs::write(&path, b"messages").unwrap();
+ let runtime = create_shard_executor().unwrap();
+ runtime.block_on(async {
+ let file = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(&path)
+ .await
+ .unwrap();
+ preallocate_file(&file, &path, RESERVATION_BYTES).await;
+ assert_eq!(
+ file.metadata().await.unwrap().len(),
+ b"messages".len() as u64
+ );
+ });
+ assert_eq!(std::fs::read(&path).unwrap(), b"messages");
+ }
+}
diff --git a/core/server_common/src/segment_storage/mod.rs b/core/server_common/src/segment_storage/mod.rs
index e000090e6..8560cd9f3 100644
--- a/core/server_common/src/segment_storage/mod.rs
+++ b/core/server_common/src/segment_storage/mod.rs
@@ -79,7 +79,7 @@ impl SegmentStorage {
.len()
== 0
{
- preallocate_file(&messages_file, Path::new(messages_path), size);
+ preallocate_file(&messages_file, Path::new(messages_path), size).await;
changed = true;
}
if changed {
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs
index 1f4c81f5c..a7dc7e7db 100644
--- a/core/shard/src/lib.rs
+++ b/core/shard/src/lib.rs
@@ -6828,7 +6828,7 @@ where
return;
}
- match planes.0.state_transfer_offer() {
+ match planes.0.state_transfer_offer().await {
Ok(offer) => {
tracing::info!(
shard = self.id,